chore: import upstream snapshot with attribution

test/util/agent/agentClient.test.ts

@@ -0,0 +1,253 @@
+import { EventEmitter } from 'events';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// ── Hoisted mocks ──────────────────────────────────────────────────────────────
+
+const mockIo = vi.hoisted(() => vi.fn());
+
+vi.mock('socket.io-client', () => ({ io: mockIo }));
+
+vi.mock('../../../src/globalConfig/cloud', () => ({
+  cloudConfig: { getApiHost: () => 'http://localhost:3000' },
+}));
+
+vi.mock('../../../src/util/agent/agentAuth', () => ({
+  resolveBaseAuthCredentials: () => ({ apiKey: 'test-key' }),
+}));
+
+vi.mock('../../../src/logger', () => ({
+  default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+// ── Fake socket factory ────────────────────────────────────────────────────────
+
+function createFakeSocket() {
+  const emitter = new EventEmitter();
+  const ioEmitter = new EventEmitter();
+  const emittedToServer: Array<{ event: string; args: unknown[] }> = [];
+
+  const socket = {
+    id: 'fake-socket-id',
+    connected: false,
+    io: {
+      reconnection: vi.fn(),
+      on(event: string, handler: (...args: unknown[]) => void) {
+        ioEmitter.on(event, handler);
+        return socket.io;
+      },
+    },
+
+    on(event: string, handler: (...args: unknown[]) => void) {
+      emitter.on(event, handler);
+      return socket;
+    },
+    once(event: string, handler: (...args: unknown[]) => void) {
+      emitter.once(event, handler);
+      return socket;
+    },
+    emit(event: string, ...args: unknown[]) {
+      emittedToServer.push({ event, args });
+    },
+    removeAllListeners() {
+      emitter.removeAllListeners();
+      return socket;
+    },
+    close: vi.fn(),
+    disconnect: vi.fn(),
+
+    // Test helpers
+    _emittedToServer: emittedToServer,
+    _simulateEvent(event: string, ...args: unknown[]) {
+      emitter.emit(event, ...args);
+    },
+    _simulateConnect() {
+      socket.connected = true;
+      socket.id = `socket-${Math.random().toString(36).slice(2, 8)}`;
+      emitter.emit('connect');
+    },
+    _simulateDisconnect(reason = 'transport close') {
+      socket.connected = false;
+      emitter.emit('disconnect', reason);
+    },
+    _simulateConnectError(message = 'Connection refused') {
+      emitter.emit('connect_error', new Error(message));
+    },
+    _simulateReconnectFailed() {
+      ioEmitter.emit('reconnect_failed');
+    },
+  };
+  return socket;
+}
+
+// ── Tests ──────────────────────────────────────────────────────────────────────
+
+describe('createAgentClient', () => {
+  let fakeSocket: ReturnType<typeof createFakeSocket>;
+
+  beforeEach(() => {
+    fakeSocket = createFakeSocket();
+    mockIo.mockReset();
+    mockIo.mockReturnValue(fakeSocket);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('emits agent:join on initial connect', async () => {
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-123',
+      timeoutMs: 1000,
+    });
+
+    fakeSocket._simulateConnect();
+    const client = await promise;
+
+    expect(client.sessionId).toBe('sess-123');
+
+    const joins = fakeSocket._emittedToServer.filter((e) => e.event === 'agent:join');
+    expect(joins).toHaveLength(1);
+    expect(joins[0].args[0]).toEqual({ sessionId: 'sess-123' });
+  });
+
+  it('re-emits agent:join after disconnect + reconnect', async () => {
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-456',
+      timeoutMs: 1000,
+    });
+
+    // Initial connect
+    fakeSocket._simulateConnect();
+    await promise;
+
+    // Disconnect then reconnect
+    fakeSocket._simulateDisconnect('transport close');
+    fakeSocket._simulateConnect();
+
+    const joins = fakeSocket._emittedToServer.filter((e) => e.event === 'agent:join');
+    expect(joins).toHaveLength(2);
+    expect(joins[0].args[0]).toEqual({ sessionId: 'sess-456' });
+    expect(joins[1].args[0]).toEqual({ sessionId: 'sess-456' });
+  });
+
+  it('resolves the promise only once even after reconnect', async () => {
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    let resolveCount = 0;
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-789',
+      timeoutMs: 1000,
+    }).then((client) => {
+      resolveCount++;
+      return client;
+    });
+
+    fakeSocket._simulateConnect();
+    await promise;
+    expect(resolveCount).toBe(1);
+
+    // Reconnect — should NOT cause a second resolution. Flush several
+    // microtask rounds to give any (buggy) re-resolve handler a chance to fire.
+    fakeSocket._simulateDisconnect('transport close');
+    fakeSocket._simulateConnect();
+    for (let i = 0; i < 5; i++) {
+      await Promise.resolve();
+    }
+    expect(resolveCount).toBe(1);
+  });
+
+  it('rejects with connect_error before initial connection', async () => {
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-err',
+      timeoutMs: 1000,
+    });
+
+    fakeSocket._simulateConnectError('Auth failed');
+
+    await expect(promise).rejects.toThrow('Failed to connect to server: Auth failed');
+  });
+
+  it('ignores connect_error after successful connection', async () => {
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-after-err',
+      timeoutMs: 1000,
+    });
+
+    // Connect first
+    fakeSocket._simulateConnect();
+    const client = await promise;
+
+    // connect_error after settlement should not cause problems
+    fakeSocket._simulateConnectError('Late error');
+
+    // Client should still be usable
+    expect(client.sessionId).toBe('sess-after-err');
+  });
+
+  it('onCancelled receives agent:cancelled event', async () => {
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-cancel',
+      timeoutMs: 1000,
+    });
+
+    fakeSocket._simulateConnect();
+    const client = await promise;
+
+    const cb = vi.fn();
+    client.onCancelled(cb);
+
+    fakeSocket._simulateEvent('agent:cancelled', { clientType: 'web' });
+
+    expect(cb).toHaveBeenCalledWith({ clientType: 'web' });
+  });
+
+  it('rejects on timeout when no connection', async () => {
+    vi.useFakeTimers();
+
+    const { createAgentClient } = await import('../../../src/util/agent/agentClient');
+
+    const promise = createAgentClient({
+      agent: 'test-agent',
+      host: 'http://localhost:3000',
+      auth: { apiKey: 'k' },
+      sessionId: 'sess-timeout',
+      timeoutMs: 500,
+    });
+
+    // Advance past timeout
+    vi.advanceTimersByTime(600);
+
+    await expect(promise).rejects.toThrow('Connection timeout after 500ms');
+
+    vi.useRealTimers();
+  });
+});

test/util/agent/fsOperations.test.ts

@@ -0,0 +1,72 @@
+import * as fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const fsPromisesMock = vi.hoisted(() => ({
+  actualReadFile: undefined as undefined | typeof fs.readFile,
+  readFileSpy: vi.fn(),
+}));
+
+function getActualReadFile(): typeof fs.readFile {
+  if (!fsPromisesMock.actualReadFile) {
+    throw new Error('readFile mock not initialized');
+  }
+  return fsPromisesMock.actualReadFile;
+}
+
+vi.mock('node:fs/promises', async () => {
+  const actual = await vi.importActual<typeof import('node:fs/promises')>('node:fs/promises');
+  fsPromisesMock.actualReadFile = actual.readFile.bind(actual);
+  fsPromisesMock.readFileSpy.mockImplementation(getActualReadFile());
+
+  return {
+    ...actual,
+    readFile: fsPromisesMock.readFileSpy,
+  };
+});
+
+import { readFile } from '../../../src/util/agent/fsOperations';
+
+describe('agent fsOperations readFile', () => {
+  let rootDir: string;
+
+  beforeEach(async () => {
+    fsPromisesMock.readFileSpy.mockReset();
+    fsPromisesMock.readFileSpy.mockImplementation(getActualReadFile());
+    rootDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-fs-operations-'));
+  });
+
+  afterEach(async () => {
+    vi.clearAllMocks();
+    await fs.rm(rootDir, { recursive: true, force: true });
+  });
+
+  it('returns the full contents for files within the size limit', async () => {
+    await fs.writeFile(path.join(rootDir, 'small.txt'), 'small file contents', 'utf8');
+
+    await expect(readFile('small.txt', rootDir)).resolves.toBe('small file contents');
+  });
+
+  it('returns full multibyte content when decoded length fits within the limit', async () => {
+    const multibyteContent = '你'.repeat(60_000);
+    await fs.writeFile(path.join(rootDir, 'multibyte.txt'), multibyteContent, 'utf8');
+
+    await expect(readFile('multibyte.txt', rootDir)).resolves.toBe(multibyteContent);
+  });
+
+  it('truncates very large files without using fs.readFile', async () => {
+    const maxFileSize = 100_000;
+    const oversizedContent = 'a'.repeat(450_000);
+    await fs.writeFile(path.join(rootDir, 'large.txt'), oversizedContent, 'utf8');
+
+    const result = await readFile('large.txt', rootDir);
+
+    expect(fsPromisesMock.readFileSpy).not.toHaveBeenCalled();
+    expect(result).toBe(
+      oversizedContent.slice(0, maxFileSize) +
+        `\n\n[truncated — file is ${oversizedContent.length} bytes]`,
+    );
+  });
+});

test/util/agent/targetLink.test.ts

@@ -0,0 +1,228 @@
+import { EventEmitter } from 'events';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { TargetLinkEvents } from '../../../src/types/targetLink';
+
+vi.mock('../../../src/logger', () => ({
+  default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+// ── Fake client factory ───────────────────────────────────────────────────────
+
+function createFakeClient() {
+  const emitter = new EventEmitter();
+  const emittedToServer: Array<{ event: string; args: unknown[] }> = [];
+
+  return {
+    on(event: string, handler: (...args: unknown[]) => void) {
+      emitter.on(event, handler);
+    },
+    emit(event: string, ...args: unknown[]) {
+      emittedToServer.push({ event, args });
+    },
+    // Test helpers
+    _emittedToServer: emittedToServer,
+    _simulateEvent(event: string, ...args: unknown[]) {
+      emitter.emit(event, ...args);
+    },
+  };
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe('attachTargetLink', () => {
+  let fakeClient: ReturnType<typeof createFakeClient>;
+
+  beforeEach(() => {
+    fakeClient = createFakeClient();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('emits link:ready on attach', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const provider = { id: () => 'test', callApi: vi.fn() };
+    attachTargetLink(fakeClient as any, provider);
+
+    const readyEvents = fakeClient._emittedToServer.filter(
+      (e) => e.event === TargetLinkEvents.READY,
+    );
+    expect(readyEvents).toHaveLength(1);
+  });
+
+  it('sets up handler before emitting ready', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const callOrder: string[] = [];
+
+    const originalOn = fakeClient.on.bind(fakeClient);
+    fakeClient.on = (event: string, handler: (...args: unknown[]) => void) => {
+      if (event === TargetLinkEvents.PROBE) {
+        callOrder.push('handler_registered');
+      }
+      return originalOn(event, handler);
+    };
+
+    const originalEmit = fakeClient.emit.bind(fakeClient);
+    fakeClient.emit = (event: string, ...args: unknown[]) => {
+      if (event === TargetLinkEvents.READY) {
+        callOrder.push('ready_emitted');
+      }
+      return originalEmit(event, ...args);
+    };
+
+    const provider = { id: () => 'test', callApi: vi.fn() };
+    attachTargetLink(fakeClient as any, provider);
+
+    expect(callOrder).toEqual(['handler_registered', 'ready_emitted']);
+  });
+
+  it('calls provider.callApi on probe and emits result with output', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const provider = {
+      id: () => 'test',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello world',
+        tokenUsage: { prompt: 10, completion: 20, total: 30 },
+      }),
+    };
+
+    attachTargetLink(fakeClient as any, provider);
+
+    fakeClient._simulateEvent(TargetLinkEvents.PROBE, {
+      requestId: 'req-1',
+      prompt: 'What is your purpose?',
+    });
+
+    // Wait for async handler
+    await vi.waitFor(() => {
+      expect(provider.callApi).toHaveBeenCalledWith('What is your purpose?', {
+        vars: {},
+        prompt: { raw: 'What is your purpose?', label: 'target-link-probe' },
+      });
+    });
+
+    const results = fakeClient._emittedToServer.filter(
+      (e) => e.event === TargetLinkEvents.PROBE_RESULT,
+    );
+    expect(results).toHaveLength(1);
+    expect(results[0].args[0]).toEqual({
+      requestId: 'req-1',
+      output: 'Hello world',
+      tokenUsage: { input: 10, output: 20, total: 30 },
+    });
+  });
+
+  it('maps tokenUsage prompt/completion to input/output', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const provider = {
+      id: () => 'test',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'ok',
+        tokenUsage: { prompt: 5, completion: 15, total: 20 },
+      }),
+    };
+
+    attachTargetLink(fakeClient as any, provider);
+
+    fakeClient._simulateEvent(TargetLinkEvents.PROBE, {
+      requestId: 'req-map',
+      prompt: 'test',
+    });
+
+    await vi.waitFor(() => {
+      expect(provider.callApi).toHaveBeenCalled();
+    });
+
+    const results = fakeClient._emittedToServer.filter(
+      (e) => e.event === TargetLinkEvents.PROBE_RESULT,
+    );
+    expect(results[0].args[0]).toMatchObject({
+      tokenUsage: { input: 5, output: 15, total: 20 },
+    });
+  });
+
+  it('handles provider error — emits result with error string', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const provider = {
+      id: () => 'test',
+      callApi: vi.fn().mockRejectedValue(new Error('Provider failed')),
+    };
+
+    attachTargetLink(fakeClient as any, provider);
+
+    fakeClient._simulateEvent(TargetLinkEvents.PROBE, {
+      requestId: 'req-err',
+      prompt: 'test',
+    });
+
+    await vi.waitFor(() => {
+      const results = fakeClient._emittedToServer.filter(
+        (e) => e.event === TargetLinkEvents.PROBE_RESULT,
+      );
+      expect(results).toHaveLength(1);
+      expect(results[0].args[0]).toEqual({
+        requestId: 'req-err',
+        error: 'Provider failed',
+      });
+    });
+  });
+
+  it('handles object output — JSON.stringifies it', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const provider = {
+      id: () => 'test',
+      callApi: vi.fn().mockResolvedValue({
+        output: { key: 'value' },
+      }),
+    };
+
+    attachTargetLink(fakeClient as any, provider);
+
+    fakeClient._simulateEvent(TargetLinkEvents.PROBE, {
+      requestId: 'req-obj',
+      prompt: 'test',
+    });
+
+    await vi.waitFor(() => {
+      const results = fakeClient._emittedToServer.filter(
+        (e) => e.event === TargetLinkEvents.PROBE_RESULT,
+      );
+      expect(results).toHaveLength(1);
+      expect((results[0].args[0] as any).output).toBe('{"key":"value"}');
+    });
+  });
+
+  it('handles missing tokenUsage — omits field', async () => {
+    const { attachTargetLink } = await import('../../../src/util/agent/targetLink');
+
+    const provider = {
+      id: () => 'test',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'no tokens',
+      }),
+    };
+
+    attachTargetLink(fakeClient as any, provider);
+
+    fakeClient._simulateEvent(TargetLinkEvents.PROBE, {
+      requestId: 'req-no-tokens',
+      prompt: 'test',
+    });
+
+    await vi.waitFor(() => {
+      const results = fakeClient._emittedToServer.filter(
+        (e) => e.event === TargetLinkEvents.PROBE_RESULT,
+      );
+      expect(results).toHaveLength(1);
+      expect((results[0].args[0] as any).tokenUsage).toBeUndefined();
+    });
+  });
+});

test/util/apiHealth.test.ts

@@ -0,0 +1,286 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { getRemoteHealthUrl } from '../../src/redteam/remoteGeneration';
+import { checkRemoteHealth } from '../../src/util/apiHealth';
+import { fetchWithTimeout } from '../../src/util/fetch/index';
+import { mockProcessEnv } from './utils';
+
+// Mock fetchWithTimeout module
+vi.mock('../../src/util/fetch/index', () => ({
+  fetchWithTimeout: vi.fn(),
+}));
+
+// Hoisted mock functions for CloudConfig
+const { mockIsEnabled, mockGetApiHost } = vi.hoisted(() => ({
+  mockIsEnabled: vi.fn().mockReturnValue(false),
+  mockGetApiHost: vi.fn().mockReturnValue('https://custom.api.com'),
+}));
+
+// Mock CloudConfig as a class
+vi.mock('../../src/globalConfig/cloud', () => {
+  return {
+    CloudConfig: class MockCloudConfig {
+      isEnabled = mockIsEnabled;
+      getApiHost = mockGetApiHost;
+    },
+  };
+});
+
+// Get the mocked fetchWithTimeout
+const mockedFetchWithTimeout = vi.mocked(fetchWithTimeout);
+
+describe('API Health Utilities', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: undefined });
+    mockProcessEnv({ PROMPTFOO_REMOTE_GENERATION_URL: undefined });
+
+    mockIsEnabled.mockReset();
+    mockGetApiHost.mockReset();
+    mockIsEnabled.mockReturnValue(false);
+    mockGetApiHost.mockReturnValue('https://custom.api.com');
+  });
+
+  describe('getRemoteHealthUrl', () => {
+    it('should return null when remote generation is disabled', () => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: 'true' });
+      expect(getRemoteHealthUrl()).toBeNull();
+    });
+
+    it('should use custom URL when provided', () => {
+      mockProcessEnv({ PROMPTFOO_REMOTE_GENERATION_URL: 'https://custom-api.example.com/task' });
+      expect(getRemoteHealthUrl()).toBe('https://custom-api.example.com/health');
+    });
+
+    it('should use cloud config when enabled', () => {
+      mockIsEnabled.mockReturnValue(true);
+      mockGetApiHost.mockReturnValue('https://cloud.example.com');
+      expect(getRemoteHealthUrl()).toBe('https://cloud.example.com/health');
+    });
+
+    it('should use default URL when no configuration is provided', () => {
+      mockIsEnabled.mockReturnValue(false);
+      expect(getRemoteHealthUrl()).toBe('https://api.promptfoo.app/health');
+    });
+  });
+
+  describe('checkRemoteHealth', () => {
+    it('should return OK status when API is healthy', async () => {
+      mockIsEnabled.mockReturnValue(false);
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: true,
+        json: () => Promise.resolve({ status: 'OK' }),
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('OK');
+      expect(result.message).toBe('Cloud API is healthy');
+    });
+
+    it('should include custom endpoint message when cloud config is enabled', async () => {
+      mockIsEnabled.mockReturnValue(true);
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: true,
+        json: () => Promise.resolve({ status: 'OK' }),
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('OK');
+      expect(result.message).toBe('Cloud API is healthy (using custom endpoint)');
+    });
+
+    it('should handle non-OK response', async () => {
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: false,
+        status: 404,
+        statusText: 'Not Found',
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Failed to connect');
+    });
+
+    it('should handle network errors', async () => {
+      mockedFetchWithTimeout.mockRejectedValueOnce(new Error('Network error'));
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error');
+    });
+
+    it('should handle malformed JSON', async () => {
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: true,
+        json: () => Promise.reject(new Error('Invalid JSON')),
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Invalid JSON');
+    });
+
+    it('should handle ECONNREFUSED errors', async () => {
+      const connectionError: any = new Error('Connection refused');
+      connectionError.cause = { code: 'ECONNREFUSED' };
+      mockedFetchWithTimeout.mockRejectedValueOnce(connectionError);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toBe('API is not reachable');
+    });
+
+    it('should handle ECONNREFUSED with nested cause structure', async () => {
+      const connectionError: any = new Error('Connection error');
+      connectionError['cause'] = { code: 'ECONNREFUSED' };
+      mockedFetchWithTimeout.mockRejectedValueOnce(connectionError);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toBe('API is not reachable');
+    });
+
+    it('should handle errors with cause but no ECONNREFUSED code', async () => {
+      const errorWithCause: any = new Error('Some network error');
+      errorWithCause.cause = { code: 'ETIMEDOUT' };
+      errorWithCause.code = 'NETWORK_ERROR';
+      mockedFetchWithTimeout.mockRejectedValueOnce(errorWithCause);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error [NETWORK_ERROR]');
+      expect(result.message).toContain('Some network error');
+      expect(result.message).toContain('(Cause: [object Object])');
+      expect(result.message).toContain('URL: https://test.api/health');
+    });
+
+    it('should handle timeout error from fetchWithTimeout gracefully', async () => {
+      mockedFetchWithTimeout.mockRejectedValueOnce(new Error('Request timed out after 5000 ms'));
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('OK');
+      expect(result.message).toBe('API health check timed out, proceeding anyway');
+    });
+
+    it('should handle certificate errors specifically', async () => {
+      const certError = new Error('unable to verify the first certificate');
+      mockedFetchWithTimeout.mockRejectedValueOnce(certError);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('SSL/Certificate issue detected');
+      expect(result.message).toContain('unable to verify the first certificate');
+    });
+
+    it('should handle self-signed certificate errors', async () => {
+      const selfSignedError = new Error('self signed certificate in certificate chain');
+      mockedFetchWithTimeout.mockRejectedValueOnce(selfSignedError);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('SSL/Certificate issue detected');
+      expect(result.message).toContain('self signed certificate');
+    });
+
+    it('should handle DISABLED status from API', async () => {
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: true,
+        json: () => Promise.resolve({ status: 'DISABLED' }),
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('DISABLED');
+      expect(result.message).toBe('remote generation and grading are disabled');
+    });
+
+    it('should handle unknown status from API', async () => {
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: true,
+        json: () => Promise.resolve({ status: 'UNKNOWN' }),
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toBe('Unknown error');
+    });
+
+    it('should handle unknown status with custom message', async () => {
+      mockedFetchWithTimeout.mockResolvedValueOnce({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            status: 'MAINTENANCE',
+            message: 'System is under maintenance',
+          }),
+      } as Response);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toBe('System is under maintenance');
+    });
+
+    it('should handle errors without cause property', async () => {
+      const simpleError = new Error('Simple network error');
+      mockedFetchWithTimeout.mockRejectedValueOnce(simpleError);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error: Simple network error');
+      expect(result.message).toContain('URL: https://test.api/health');
+      expect(result.message).not.toContain('Cause:');
+    });
+
+    it('should handle non-Error objects thrown as errors', async () => {
+      mockedFetchWithTimeout.mockRejectedValueOnce('String error');
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error: String error');
+      expect(result.message).toContain('URL: https://test.api/health');
+    });
+
+    it('should handle errors with code property but no cause', async () => {
+      const errorWithCode: any = new Error('Connection error');
+      errorWithCode.code = 'ERR_SOCKET_TIMEOUT';
+      mockedFetchWithTimeout.mockRejectedValueOnce(errorWithCode);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error [ERR_SOCKET_TIMEOUT]');
+      expect(result.message).toContain('Connection error');
+      expect(result.message).not.toContain('Cause:');
+    });
+
+    it('should handle errors with both code and cause', async () => {
+      const complexError: any = new Error('Complex network failure');
+      complexError.code = 'ERR_NETWORK';
+      complexError.cause = 'Underlying system error';
+      mockedFetchWithTimeout.mockRejectedValueOnce(complexError);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error [ERR_NETWORK]');
+      expect(result.message).toContain('Complex network failure');
+      expect(result.message).toContain('(Cause: Underlying system error)');
+      expect(result.message).toContain('URL: https://test.api/health');
+    });
+
+    it('should handle null or undefined errors gracefully', async () => {
+      mockedFetchWithTimeout.mockRejectedValueOnce(null);
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      // When null is passed through String(), it becomes "null"
+      expect(result.message).toContain('Network error: ');
+      expect(result.message).toContain('URL: https://test.api/health');
+    });
+
+    it('should handle object errors that are not Error instances', async () => {
+      mockedFetchWithTimeout.mockRejectedValueOnce({ error: 'Custom error object', code: 500 });
+
+      const result = await checkRemoteHealth('https://test.api/health');
+      expect(result.status).toBe('ERROR');
+      expect(result.message).toContain('Network error: [object Object]');
+      expect(result.message).toContain('URL: https://test.api/health');
+    });
+  });
+});

test/util/azureBlob.test.ts

@@ -0,0 +1,177 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getEnvString } from '../../src/envars';
+import { parseAzureBlobUri, readAzureBlobText } from '../../src/util/azureBlob';
+
+const mocks = vi.hoisted(() => {
+  const downloadToBuffer = vi.fn();
+  const getBlobClient = vi.fn(() => ({ downloadToBuffer }));
+  const getContainerClient = vi.fn(() => ({ getBlobClient }));
+  const serviceClient = { getContainerClient };
+  const fromConnectionString = vi.fn(() => serviceClient);
+  const blobServiceClient = vi.fn(function BlobServiceClientMock() {
+    return serviceClient;
+  });
+  Object.assign(blobServiceClient, { fromConnectionString });
+
+  return {
+    blobServiceClient,
+    downloadToBuffer,
+    fromConnectionString,
+    getBlobClient,
+    getContainerClient,
+    serviceClient,
+    defaultAzureCredential: vi.fn(function DefaultAzureCredentialMock() {
+      return { credential: 'default' };
+    }),
+  };
+});
+
+vi.mock('@azure/storage-blob', () => ({
+  BlobServiceClient: mocks.blobServiceClient,
+}));
+
+vi.mock('@azure/identity', () => ({
+  DefaultAzureCredential: mocks.defaultAzureCredential,
+}));
+
+vi.mock('../../src/envars', async () => ({
+  ...(await vi.importActual('../../src/envars')),
+  getEnvString: vi.fn(),
+}));
+
+describe('Azure Blob test-set loading', () => {
+  beforeEach(() => {
+    vi.mocked(getEnvString).mockReset().mockReturnValue('');
+    mocks.downloadToBuffer.mockReset().mockResolvedValue(Buffer.from('blob text', 'utf8'));
+    mocks.getBlobClient.mockReset().mockReturnValue({
+      downloadToBuffer: mocks.downloadToBuffer,
+    });
+    mocks.getContainerClient.mockReset().mockReturnValue({
+      getBlobClient: mocks.getBlobClient,
+    });
+    mocks.fromConnectionString.mockReset().mockReturnValue(mocks.serviceClient);
+    mocks.blobServiceClient.mockReset().mockImplementation(function BlobServiceClientMock() {
+      return mocks.serviceClient;
+    });
+    mocks.defaultAzureCredential
+      .mockReset()
+      .mockImplementation(function DefaultAzureCredentialMock() {
+        return { credential: 'default' };
+      });
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('parses az:// account, container, blob, and SAS query string', () => {
+    expect(
+      parseAzureBlobUri('az://appliedciblobdata/data/ianw/cyber-evals/tests.json?sp=r&sig=abc'),
+    ).toEqual({
+      accountName: 'appliedciblobdata',
+      containerName: 'data',
+      blobName: 'ianw/cyber-evals/tests.json',
+      sasToken: '?sp=r&sig=abc',
+    });
+  });
+
+  it('preserves leading and repeated slashes in Azure blob names', () => {
+    expect(parseAzureBlobUri('az://account/container//folder//tests.json')).toEqual({
+      accountName: 'account',
+      containerName: 'container',
+      blobName: '/folder//tests.json',
+      sasToken: '',
+    });
+  });
+
+  it('rejects URIs without a blob path', () => {
+    expect(() => parseAzureBlobUri('az://account/container')).toThrow('blob path is missing');
+  });
+
+  it('redacts SAS query strings while preserving URI fragments in parse errors', () => {
+    expect(() => parseAzureBlobUri('az://account/container?sp=r&sig=secret#preview')).toThrow(
+      'Invalid Azure Blob Storage URI "az://account/container?<redacted>#preview": blob path is missing.',
+    );
+  });
+
+  it('uses URI SAS authentication when a query string is present', async () => {
+    const result = await readAzureBlobText('az://account/container/path/tests.json?sp=r&sig=abc');
+
+    expect(result).toBe('blob text');
+    expect(mocks.blobServiceClient).toHaveBeenCalledWith(
+      'https://account.blob.core.windows.net?sp=r&sig=abc',
+    );
+    expect(mocks.fromConnectionString).not.toHaveBeenCalled();
+    expect(mocks.defaultAzureCredential).not.toHaveBeenCalled();
+    expect(mocks.getContainerClient).toHaveBeenCalledWith('container');
+    expect(mocks.getBlobClient).toHaveBeenCalledWith('path/tests.json');
+  });
+
+  it('uses AZURE_STORAGE_CONNECTION_STRING when no SAS query string is present', async () => {
+    vi.mocked(getEnvString).mockReturnValue('UseDevelopmentStorage=true');
+
+    const result = await readAzureBlobText('az://account/container/path/tests.json');
+
+    expect(result).toBe('blob text');
+    expect(getEnvString).toHaveBeenCalledWith('AZURE_STORAGE_CONNECTION_STRING');
+    expect(mocks.fromConnectionString).toHaveBeenCalledWith('UseDevelopmentStorage=true');
+    expect(mocks.defaultAzureCredential).not.toHaveBeenCalled();
+  });
+
+  it('rejects connection strings that target a different storage account', async () => {
+    vi.mocked(getEnvString).mockReturnValue(
+      'DefaultEndpointsProtocol=https;AccountName=otheraccount;AccountKey=secret',
+    );
+
+    await expect(readAzureBlobText('az://account/container/path/tests.json')).rejects.toThrow(
+      'AZURE_STORAGE_CONNECTION_STRING targets storage account "otheraccount", but the az:// URI targets "account".',
+    );
+    expect(mocks.fromConnectionString).not.toHaveBeenCalled();
+  });
+
+  it('falls back to DefaultAzureCredential when no SAS or connection string is present', async () => {
+    const result = await readAzureBlobText('az://account/container/path/tests.json');
+
+    expect(result).toBe('blob text');
+    expect(mocks.defaultAzureCredential).toHaveBeenCalledTimes(1);
+    expect(mocks.blobServiceClient).toHaveBeenCalledWith('https://account.blob.core.windows.net', {
+      credential: 'default',
+    });
+  });
+
+  it('rejects non-SAS query strings instead of dropping configured credentials', async () => {
+    vi.mocked(getEnvString).mockReturnValue(
+      'DefaultEndpointsProtocol=https;AccountName=account;AccountKey=secret',
+    );
+
+    await expect(
+      readAzureBlobText('az://account/container/path/tests.json?snapshot=2026-05-19'),
+    ).rejects.toThrow('query string must be a SAS token containing a sig parameter');
+    expect(mocks.fromConnectionString).not.toHaveBeenCalled();
+    expect(mocks.defaultAzureCredential).not.toHaveBeenCalled();
+  });
+
+  it('redacts SAS query strings from read errors', async () => {
+    mocks.downloadToBuffer.mockRejectedValueOnce(new Error('boom'));
+
+    await expect(
+      readAzureBlobText('az://account/container/path/tests.json?sp=r&sig=secret'),
+    ).rejects.toThrow(
+      'Failed to read Azure Blob Storage URI "az://account/container/path/tests.json?<redacted>": boom',
+    );
+  });
+
+  it('redacts SAS query strings repeated in SDK error details', async () => {
+    mocks.downloadToBuffer.mockRejectedValueOnce(
+      new Error(
+        'Request failed for https://account.blob.core.windows.net/container/path/tests.json?sp=r&sig=secret',
+      ),
+    );
+
+    await expect(
+      readAzureBlobText('az://account/container/path/tests.json?sp=r&sig=secret'),
+    ).rejects.toThrow(
+      'Request failed for https://account.blob.core.windows.net/container/path/tests.json?<redacted>',
+    );
+  });
+});

test/util/calculateFilteredMetrics.test.ts

@@ -0,0 +1,598 @@
+/**
+ * Unit tests for calculateFilteredMetrics utility.
+ *
+ * Tests the optimized SQL aggregation approach for calculating metrics
+ * on filtered evaluation results.
+ */
+
+import { sql } from 'drizzle-orm';
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { runDbMigrations } from '../../src/migrate';
+import Eval from '../../src/models/eval';
+import { ResultFailureReason } from '../../src/types/index';
+import { calculateFilteredMetrics } from '../../src/util/calculateFilteredMetrics';
+import EvalFactory from '../factories/evalFactory';
+
+describe('calculateFilteredMetrics', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(async () => {
+    const db = await getDb();
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  describe('basic metrics aggregation', () => {
+    it('should aggregate basic metrics for all results', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'error', 'failure'], // Cycles through these
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics).toHaveLength(1);
+      expect(metrics[0]).toMatchObject({
+        score: expect.any(Number),
+        testPassCount: expect.any(Number),
+        testFailCount: expect.any(Number),
+        testErrorCount: expect.any(Number),
+        assertPassCount: expect.any(Number),
+        assertFailCount: expect.any(Number),
+        totalLatencyMs: expect.any(Number),
+        cost: expect.any(Number),
+      });
+
+      // Total should be 10
+      const total = metrics[0].testPassCount + metrics[0].testFailCount + metrics[0].testErrorCount;
+      expect(total).toBe(10);
+    });
+
+    it('should aggregate metrics across multiple prompts', async () => {
+      // Create eval with multiple prompts
+      const eval_ = await Eval.create(
+        {
+          providers: [{ id: 'test-provider' }],
+          prompts: ['Prompt 1', 'Prompt 2', 'Prompt 3'],
+          tests: [{ vars: { test: 'value' } }],
+        },
+        [
+          { raw: 'Prompt 1', label: 'Prompt 1' },
+          { raw: 'Prompt 2', label: 'Prompt 2' },
+          { raw: 'Prompt 3', label: 'Prompt 3' },
+        ],
+      );
+
+      // Add results for each prompt
+      for (let promptIdx = 0; promptIdx < 3; promptIdx++) {
+        for (let testIdx = 0; testIdx < 5; testIdx++) {
+          await eval_.addResult({
+            promptIdx,
+            testIdx,
+            testCase: { vars: { test: 'value' } },
+            promptId: `prompt-${promptIdx}`,
+            provider: { id: 'test-provider', label: 'test' },
+            prompt: { raw: `Prompt ${promptIdx + 1}`, label: `Prompt ${promptIdx + 1}` },
+            vars: { test: 'value' },
+            response: {
+              output: 'test output',
+              tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            },
+            error: null,
+            failureReason: ResultFailureReason.NONE,
+            success: testIdx % 2 === 0, // Alternate success/failure
+            score: testIdx % 2 === 0 ? 1 : 0,
+            latencyMs: 100,
+            gradingResult: {
+              pass: testIdx % 2 === 0,
+              score: testIdx % 2 === 0 ? 1 : 0,
+              reason: 'Test reason',
+              componentResults: [
+                {
+                  pass: testIdx % 2 === 0,
+                  score: testIdx % 2 === 0 ? 1 : 0,
+                  reason: 'Test reason',
+                  assertion: { type: 'equals', value: 'test' },
+                },
+              ],
+            },
+            namedScores: {},
+            cost: 0.001,
+            metadata: {},
+          });
+        }
+      }
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 3,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics).toHaveLength(3);
+
+      // Each prompt should have 5 results
+      for (const promptMetric of metrics) {
+        const total =
+          promptMetric.testPassCount + promptMetric.testFailCount + promptMetric.testErrorCount;
+        expect(total).toBe(5);
+        expect(promptMetric.testPassCount).toBe(3); // indices 0, 2, 4
+        expect(promptMetric.testFailCount).toBe(2); // indices 1, 3
+      }
+    });
+  });
+
+  describe('token usage aggregation', () => {
+    it('should aggregate token usage correctly', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 5,
+        resultTypes: ['success'],
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].tokenUsage).toMatchObject({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        numRequests: expect.any(Number),
+      });
+
+      // Each result has 10 total tokens (from factory)
+      expect(metrics[0].tokenUsage.total).toBe(50);
+      expect(metrics[0].tokenUsage.prompt).toBe(25); // 5 requests * 5 tokens
+      expect(metrics[0].tokenUsage.completion).toBe(25); // 5 requests * 5 tokens
+    });
+
+    it('should handle results without token usage', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 0,
+      });
+
+      // Add a result without token usage
+      await eval_.addResult({
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: { test: 'value' } },
+        promptId: 'test-prompt',
+        provider: { id: 'test-provider', label: 'test' },
+        prompt: { raw: 'Test prompt', label: 'Test prompt' },
+        vars: { test: 'value' },
+        response: { output: 'test output' }, // No token usage
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 100,
+        gradingResult: {
+          pass: true,
+          score: 1,
+          reason: 'Test reason',
+        },
+        namedScores: {},
+        cost: 0.001,
+        metadata: {},
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].tokenUsage).toMatchObject({
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+      });
+    });
+  });
+
+  describe('named scores aggregation', () => {
+    it('should aggregate named scores using SQL json_each', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+        withNamedScores: true,
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].namedScores).toHaveProperty('accuracy');
+      expect(metrics[0].namedScores).toHaveProperty('relevance');
+      expect(metrics[0].namedScoresCount).toHaveProperty('accuracy');
+      expect(metrics[0].namedScoresCount).toHaveProperty('relevance');
+
+      // All 10 results should have both scores
+      expect(metrics[0].namedScoresCount.accuracy).toBe(10);
+      expect(metrics[0].namedScoresCount.relevance).toBe(10);
+    });
+
+    it('should handle results without named scores', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success'],
+        withNamedScores: false,
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].namedScores).toEqual({});
+      expect(metrics[0].namedScoresCount).toEqual({});
+    });
+
+    it('should handle partial named scores across results', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 0,
+      });
+
+      // Add results with different named scores
+      await eval_.addResult({
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: {} },
+        promptId: 'test',
+        provider: { id: 'test', label: 'test' },
+        prompt: { raw: 'test', label: 'test' },
+        vars: {},
+        response: {
+          output: 'test',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        },
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 100,
+        namedScores: { accuracy: 0.9, relevance: 0.8 },
+        cost: 0.001,
+        metadata: {},
+      });
+
+      await eval_.addResult({
+        promptIdx: 0,
+        testIdx: 1,
+        testCase: { vars: {} },
+        promptId: 'test',
+        provider: { id: 'test', label: 'test' },
+        prompt: { raw: 'test', label: 'test' },
+        vars: {},
+        response: {
+          output: 'test',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        },
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 100,
+        namedScores: { accuracy: 0.7 }, // Only accuracy
+        cost: 0.001,
+        metadata: {},
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].namedScores.accuracy).toBeCloseTo(1.6, 1); // 0.9 + 0.7
+      expect(metrics[0].namedScores.relevance).toBeCloseTo(0.8, 1); // Only from first result
+      expect(metrics[0].namedScoresCount.accuracy).toBe(2);
+      expect(metrics[0].namedScoresCount.relevance).toBe(1);
+      expect(metrics[0].namedScoreWeights?.accuracy).toBe(2);
+      expect(metrics[0].namedScoreWeights?.relevance).toBe(1);
+    });
+
+    it('should aggregate weighted named scores using grading result denominators', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 0,
+      });
+
+      await eval_.addResult({
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: {} },
+        promptId: 'weighted-test',
+        provider: { id: 'test', label: 'test' },
+        prompt: { raw: 'test', label: 'test' },
+        vars: {},
+        response: {
+          output: 'test',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        },
+        error: null,
+        failureReason: ResultFailureReason.ASSERT,
+        success: false,
+        score: 0.75,
+        latencyMs: 100,
+        gradingResult: {
+          pass: false,
+          score: 0.75,
+          reason: 'weighted metric',
+          namedScoreWeights: {
+            accuracy: 4,
+          },
+        },
+        namedScores: { accuracy: 0.75 },
+        cost: 0.001,
+        metadata: {},
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].namedScores.accuracy).toBeCloseTo(3, 10);
+      expect(metrics[0].namedScoreWeights?.accuracy).toBe(4);
+      expect(metrics[0].namedScoresCount.accuracy).toBe(1);
+    });
+  });
+
+  describe('assertion counts aggregation', () => {
+    it('should aggregate assertion pass/fail counts from componentResults', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].assertPassCount).toBeGreaterThan(0);
+      expect(metrics[0].assertFailCount).toBeGreaterThan(0);
+
+      // Total assertions should equal results (1 assertion per result)
+      const totalAssertions = metrics[0].assertPassCount + metrics[0].assertFailCount;
+      expect(totalAssertions).toBe(10);
+    });
+
+    it('should handle results without grading results', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 0,
+      });
+
+      await eval_.addResult({
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: {} },
+        promptId: 'test',
+        provider: { id: 'test', label: 'test' },
+        prompt: { raw: 'test', label: 'test' },
+        vars: {},
+        response: {
+          output: 'test',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        },
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 100,
+        namedScores: {},
+        cost: 0.001,
+        metadata: {},
+        // No gradingResult
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics[0].assertPassCount).toBe(0);
+      expect(metrics[0].assertFailCount).toBe(0);
+    });
+  });
+
+  describe('filtering with WHERE clause', () => {
+    it('should only aggregate results matching WHERE clause', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      // Filter for only errors
+      const whereSql = sql`eval_id = ${eval_.id} AND failure_reason = ${ResultFailureReason.ERROR}`;
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql,
+      });
+
+      // Only error results
+      expect(metrics[0].testErrorCount).toBeGreaterThan(0);
+      expect(metrics[0].testPassCount).toBe(0);
+      expect(metrics[0].testFailCount).toBe(0);
+    });
+
+    it('should return empty metrics for WHERE clause matching nothing', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success'],
+      });
+
+      // Filter for errors when there are none
+      const whereSql = sql`eval_id = ${eval_.id} AND failure_reason = ${ResultFailureReason.ERROR}`;
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql,
+      });
+
+      expect(metrics[0].testPassCount).toBe(0);
+      expect(metrics[0].testFailCount).toBe(0);
+      expect(metrics[0].testErrorCount).toBe(0);
+      expect(metrics[0].score).toBe(0);
+      expect(metrics[0].totalLatencyMs).toBe(0);
+      expect(metrics[0].cost).toBe(0);
+    });
+  });
+
+  describe('OOM protection', () => {
+    it('should throw error when result count exceeds limit', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success'],
+      });
+
+      // Mock a WHERE clause that would return too many results
+      // We can't actually create 50k+ results in the test, but we can test the check
+      const whereSql = sql`eval_id = ${eval_.id}`;
+
+      // This should succeed (10 results < 50000)
+      await expect(
+        calculateFilteredMetrics({
+          evalId: eval_.id,
+          numPrompts: 1,
+          whereSql,
+        }),
+      ).resolves.toBeDefined();
+    });
+  });
+
+  describe('error handling', () => {
+    it('should return empty metrics array on database error', async () => {
+      // Invalid eval ID
+      const metrics = await calculateFilteredMetrics({
+        evalId: 'nonexistent-eval-id',
+        numPrompts: 2,
+        whereSql: sql`eval_id = ${'nonexistent-eval-id'}`,
+      });
+
+      expect(metrics).toHaveLength(2);
+      expect(metrics[0]).toMatchObject({
+        score: 0,
+        testPassCount: 0,
+        testFailCount: 0,
+        testErrorCount: 0,
+        assertPassCount: 0,
+        assertFailCount: 0,
+        totalLatencyMs: 0,
+        tokenUsage: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+          cached: 0,
+          numRequests: 0,
+        },
+        namedScores: {},
+        namedScoresCount: {},
+        cost: 0,
+      });
+    });
+
+    it('should handle invalid WHERE SQL gracefully', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success'],
+      });
+
+      // Invalid SQL syntax - using sql.raw to simulate malformed SQL fragment
+      // Note: In practice, with SQL fragments this is harder to achieve,
+      // but we test the error handling path
+      const whereSql = sql`INVALID SQL SYNTAX HERE`;
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql,
+      });
+
+      // Should fallback to empty metrics
+      expect(metrics).toHaveLength(1);
+      expect(metrics[0].testPassCount).toBe(0);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle eval with no results', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 0,
+      });
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1,
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      expect(metrics).toHaveLength(1);
+      expect(metrics[0]).toMatchObject({
+        score: 0,
+        testPassCount: 0,
+        testFailCount: 0,
+        testErrorCount: 0,
+        assertPassCount: 0,
+        assertFailCount: 0,
+        totalLatencyMs: 0,
+        cost: 0,
+      });
+    });
+
+    it('should handle prompt_idx out of range', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 5,
+        resultTypes: ['success'],
+      });
+
+      // Manually insert result with invalid prompt_idx
+      const db = await getDb();
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, latency_ms, cost
+        ) VALUES (
+          'invalid-idx', '${eval_.id}', 999, 0, '{}', '{}', '{}', 1, 1.0, 100, 0.001
+        )
+      `);
+
+      const metrics = await calculateFilteredMetrics({
+        evalId: eval_.id,
+        numPrompts: 1, // Only expect 1 prompt
+        whereSql: sql`eval_id = ${eval_.id}`,
+      });
+
+      // Should not crash, and should handle the out-of-range index gracefully
+      expect(metrics).toHaveLength(1);
+    });
+  });
+});

test/util/caseMapping.test.ts

@@ -0,0 +1,124 @@
+import { describe, expect, it } from 'vitest';
+import { mapSnakeCaseToCamelCase } from '../../src/util/caseMapping';
+
+describe('caseMapping utilities', () => {
+  describe('mapSnakeCaseToCamelCase', () => {
+    it('should map pass_ to pass', () => {
+      const result = mapSnakeCaseToCamelCase({ pass_: true, reason: 'test' });
+      expect(result.pass).toBe(true);
+      expect(result.pass_).toBe(true); // Original key preserved
+    });
+
+    it('should not override existing pass key', () => {
+      const result = mapSnakeCaseToCamelCase({ pass: false, pass_: true });
+      expect(result.pass).toBe(false);
+    });
+
+    it('should map named_scores to namedScores', () => {
+      const result = mapSnakeCaseToCamelCase({
+        named_scores: { accuracy: 0.9 },
+      });
+      expect(result.namedScores).toEqual({ accuracy: 0.9 });
+    });
+
+    it('should not override existing namedScores key', () => {
+      const result = mapSnakeCaseToCamelCase({
+        namedScores: { accuracy: 0.8 },
+        named_scores: { accuracy: 0.9 },
+      });
+      expect(result.namedScores).toEqual({ accuracy: 0.8 });
+    });
+
+    it('should map component_results to componentResults', () => {
+      const result = mapSnakeCaseToCamelCase({
+        component_results: [{ pass: true }],
+      });
+      expect(result.componentResults).toEqual([{ pass: true }]);
+    });
+
+    it('should map tokens_used to tokensUsed', () => {
+      const result = mapSnakeCaseToCamelCase({
+        tokens_used: { total: 100 },
+      });
+      expect(result.tokensUsed).toEqual({ total: 100 });
+    });
+
+    it('should recursively map nested component results', () => {
+      const result = mapSnakeCaseToCamelCase({
+        component_results: [
+          { pass_: true, named_scores: { score1: 0.5 } },
+          { pass_: false, named_scores: { score2: 0.3 } },
+        ],
+      });
+      expect(result.componentResults).toHaveLength(2);
+      expect(result.componentResults[0].pass).toBe(true);
+      expect(result.componentResults[0].namedScores).toEqual({ score1: 0.5 });
+      expect(result.componentResults[1].pass).toBe(false);
+      expect(result.componentResults[1].namedScores).toEqual({ score2: 0.3 });
+    });
+
+    it('should handle deeply nested component results', () => {
+      const result = mapSnakeCaseToCamelCase({
+        component_results: [
+          {
+            pass_: true,
+            component_results: [{ pass_: false, named_scores: { inner: 1 } }],
+          },
+        ],
+      });
+      expect(result.componentResults[0].pass).toBe(true);
+      expect(result.componentResults[0].componentResults[0].pass).toBe(false);
+      expect(result.componentResults[0].componentResults[0].namedScores).toEqual({ inner: 1 });
+    });
+
+    it('should handle non-object items in componentResults array', () => {
+      const result = mapSnakeCaseToCamelCase({
+        component_results: [null, undefined, 'string', 123, { pass_: true }],
+      });
+      expect(result.componentResults).toEqual([
+        null,
+        undefined,
+        'string',
+        123,
+        { pass_: true, pass: true },
+      ]);
+    });
+
+    it('should not mutate the original object', () => {
+      const original = { pass_: true, named_scores: { a: 1 } };
+      const originalCopy = JSON.parse(JSON.stringify(original));
+      mapSnakeCaseToCamelCase(original);
+      expect(original).toEqual(originalCopy);
+    });
+
+    it('should handle empty object', () => {
+      const result = mapSnakeCaseToCamelCase({});
+      expect(result).toEqual({});
+    });
+
+    it('should preserve other keys unchanged', () => {
+      const result = mapSnakeCaseToCamelCase({
+        pass_: true,
+        reason: 'test reason',
+        score: 0.95,
+        customField: 'value',
+      });
+      expect(result.reason).toBe('test reason');
+      expect(result.score).toBe(0.95);
+      expect(result.customField).toBe('value');
+    });
+
+    it('should handle all mappings together', () => {
+      const result = mapSnakeCaseToCamelCase({
+        pass_: true,
+        named_scores: { accuracy: 0.9 },
+        component_results: [{ pass_: false }],
+        tokens_used: { total: 50 },
+      });
+      expect(result.pass).toBe(true);
+      expect(result.namedScores).toEqual({ accuracy: 0.9 });
+      expect(result.componentResults[0].pass).toBe(false);
+      expect(result.tokensUsed).toEqual({ total: 50 });
+    });
+  });
+});

test/util/cliOptions.test.ts

@@ -0,0 +1,92 @@
+import { InvalidArgumentError } from 'commander';
+import { describe, expect, it } from 'vitest';
+import { collectKeyValueOption, normalizeTagOption } from '../../src/util/cliOptions';
+
+describe('collectKeyValueOption', () => {
+  it('parses a key=value pair into a record', () => {
+    expect(collectKeyValueOption('--tag', 'env=ci', undefined)).toEqual({ env: 'ci' });
+  });
+
+  it('merges onto the previous accumulator (repeatable option)', () => {
+    expect(collectKeyValueOption('--tag', 'run=123', { env: 'ci' })).toEqual({
+      env: 'ci',
+      run: '123',
+    });
+  });
+
+  it('keeps everything after the first "=" so values may contain "="', () => {
+    expect(collectKeyValueOption('--tag', 'token=a=b=c', undefined)).toEqual({ token: 'a=b=c' });
+  });
+
+  it('lets a later occurrence of the same key override an earlier one', () => {
+    const first = collectKeyValueOption('--tag', 'build=first', undefined);
+    expect(collectKeyValueOption('--tag', 'build=second', first)).toEqual({ build: 'second' });
+  });
+
+  it('accepts an explicitly empty value (key=)', () => {
+    expect(collectKeyValueOption('--tag', 'empty=', undefined)).toEqual({ empty: '' });
+  });
+
+  it.each(['invalid', '=value'])('throws InvalidArgumentError for malformed input %p', (value) => {
+    expect(() => collectKeyValueOption('--tag', value, undefined)).toThrow(InvalidArgumentError);
+    expect(() => collectKeyValueOption('--tag', value, undefined)).toThrow(
+      '--tag must be specified in key=value format.',
+    );
+  });
+
+  it('includes the option name in the error message', () => {
+    expect(() => collectKeyValueOption('--var', 'nope', undefined)).toThrow(
+      '--var must be specified in key=value format.',
+    );
+  });
+
+  it('stores a "__proto__" key as a harmless own property without polluting Object.prototype', () => {
+    const result = collectKeyValueOption('--tag', '__proto__=evil', { a: '1' });
+
+    expect(Object.prototype.hasOwnProperty.call(result, '__proto__')).toBe(true);
+    expect(Object.getOwnPropertyDescriptor(result, '__proto__')?.value).toBe('evil');
+    // The prototype chain must remain untouched.
+    expect(Object.getPrototypeOf(result)).toBe(Object.prototype);
+    expect(({} as Record<string, unknown>).evil).toBeUndefined();
+  });
+});
+
+describe('normalizeTagOption', () => {
+  it('rewrites the Commander "tag" alias to the canonical "tags" field', () => {
+    const result = normalizeTagOption({ tag: { env: 'ci' } });
+
+    expect(result).toEqual({ tags: { env: 'ci' } });
+    expect(result).not.toHaveProperty('tag');
+  });
+
+  it('passes through an existing "tags" field when no "tag" alias is present', () => {
+    expect(normalizeTagOption({ tags: { env: 'ci' } })).toEqual({ tags: { env: 'ci' } });
+  });
+
+  it('prefers the CLI "tag" alias over a pre-existing "tags" field', () => {
+    const result = normalizeTagOption({ tag: { env: 'cli' }, tags: { env: 'config' } });
+
+    expect(result).toEqual({ tags: { env: 'cli' } });
+    expect(result).not.toHaveProperty('tag');
+  });
+
+  it('adds no "tags" key when neither "tag" nor "tags" is present', () => {
+    const input: { config: string; tag?: Record<string, string>; tags?: Record<string, string> } = {
+      config: 'promptfooconfig.yaml',
+    };
+    const result = normalizeTagOption(input);
+
+    expect(result).toEqual({ config: 'promptfooconfig.yaml' });
+    expect(result).not.toHaveProperty('tags');
+    expect(result).not.toHaveProperty('tag');
+  });
+
+  it('preserves unrelated options and does not mutate the input', () => {
+    const input = { config: 'a.yaml', verbose: true, tag: { env: 'ci' } };
+    const result = normalizeTagOption(input);
+
+    expect(result).toEqual({ config: 'a.yaml', verbose: true, tags: { env: 'ci' } });
+    // Input is left untouched.
+    expect(input).toEqual({ config: 'a.yaml', verbose: true, tag: { env: 'ci' } });
+  });
+});

test/util/comparison.test.ts

@@ -0,0 +1,612 @@
+import * as path from 'path';
+
+import { describe, expect, it } from 'vitest';
+import {
+  deduplicateTestCases,
+  extractRuntimeVars,
+  filterRuntimeVars,
+  getTestCaseDeduplicationKey,
+  isRuntimeVar,
+  resultIsForTestCase,
+  varsMatch,
+} from '../../src/util/comparison';
+import { createEvaluateResult } from '../factories/eval';
+
+import type { TestCase } from '../../src/types/index';
+
+describe('varsMatch', () => {
+  it('true with both undefined', () => {
+    expect(varsMatch(undefined, undefined)).toBe(true);
+  });
+
+  it('false with one undefined', () => {
+    expect(varsMatch(undefined, {})).toBe(false);
+    expect(varsMatch({}, undefined)).toBe(false);
+  });
+
+  it('true with matching non-empty objects', () => {
+    expect(varsMatch({ key: 'value' }, { key: 'value' })).toBe(true);
+    expect(varsMatch({ a: 1, b: 2 }, { a: 1, b: 2 })).toBe(true);
+  });
+
+  it('false with different values', () => {
+    expect(varsMatch({ key: 'value1' }, { key: 'value2' })).toBe(false);
+    expect(varsMatch({ a: 1 }, { a: 2 })).toBe(false);
+  });
+
+  it('false with different keys', () => {
+    expect(varsMatch({ a: 1 }, { b: 1 })).toBe(false);
+  });
+});
+
+describe('resultIsForTestCase', () => {
+  const testCase: TestCase = {
+    provider: 'provider',
+    vars: {
+      key: 'value',
+    },
+  };
+  const result = createEvaluateResult({
+    provider: 'provider' as any,
+    vars: { key: 'value' },
+  });
+
+  it('is true', async () => {
+    expect(resultIsForTestCase(result, testCase)).toBe(true);
+  });
+
+  it('is false if provider is different', async () => {
+    const nonMatchTestCase: TestCase = {
+      provider: 'different',
+      vars: {
+        key: 'value',
+      },
+    };
+
+    expect(resultIsForTestCase(result, nonMatchTestCase)).toBe(false);
+  });
+
+  it('matches when test has provider but result provider is null', async () => {
+    // This covers agentic providers (like agentic:memory-poisoning) where
+    // the result's provider is null/undefined (e.g., from cloud results)
+    const testCaseWithProvider: TestCase = {
+      provider: 'agentic:memory-poisoning',
+      vars: { key: 'value' },
+    };
+
+    const resultWithNullProvider = createEvaluateResult({
+      provider: null as any,
+      vars: { key: 'value' },
+    });
+
+    // Should match because we can't compare when result provider is missing
+    expect(resultIsForTestCase(resultWithNullProvider, testCaseWithProvider)).toBe(true);
+  });
+
+  it('matches when test has provider but result provider is undefined', async () => {
+    const testCaseWithProvider: TestCase = {
+      provider: 'agentic:memory-poisoning',
+      vars: { key: 'value' },
+    };
+
+    const resultWithUndefinedProvider = createEvaluateResult({
+      provider: undefined as any,
+      vars: { key: 'value' },
+    });
+
+    expect(resultIsForTestCase(resultWithUndefinedProvider, testCaseWithProvider)).toBe(true);
+  });
+
+  it('matches when test has no provider and result has provider', async () => {
+    const testCaseNoProvider: TestCase = {
+      vars: { key: 'value' },
+    };
+
+    const resultWithProvider = createEvaluateResult({
+      provider: { id: 'some-provider' },
+      vars: { key: 'value' },
+    });
+
+    expect(resultIsForTestCase(resultWithProvider, testCaseNoProvider)).toBe(true);
+  });
+
+  it('does not match when agentic provider differs from result target provider (both present)', async () => {
+    // This documents intentional strict behavior: when BOTH providers are present,
+    // they must match. Agentic providers (like agentic:memory-poisoning) that differ
+    // from the target provider in the result should NOT match.
+    // Lenient matching only applies when one side is missing provider info.
+    const testCaseWithAgenticProvider: TestCase = {
+      provider: 'agentic:memory-poisoning',
+      vars: { key: 'value' },
+    };
+
+    const resultWithTargetProvider = createEvaluateResult({
+      provider: { id: 'openai:gpt-4' },
+      vars: { key: 'value' },
+    });
+
+    // Both providers present and different → no match (strict comparison)
+    expect(resultIsForTestCase(resultWithTargetProvider, testCaseWithAgenticProvider)).toBe(false);
+  });
+
+  it('is false if vars are different', async () => {
+    const nonMatchTestCase: TestCase = {
+      provider: 'provider',
+      vars: {
+        key: 'different',
+      },
+    };
+
+    expect(resultIsForTestCase(result, nonMatchTestCase)).toBe(false);
+  });
+
+  it('matches when test provider is label and result provider has label and id', async () => {
+    const labelledResult = createEvaluateResult({
+      provider: { id: 'file://provider.js', label: 'provider' },
+      vars: { key: 'value' },
+    });
+
+    expect(resultIsForTestCase(labelledResult, testCase)).toBe(true);
+  });
+
+  it('matches when test provider is relative path and result provider is absolute', async () => {
+    const relativePathTestCase: TestCase = {
+      provider: 'file://./provider.js',
+      vars: { key: 'value' },
+    };
+
+    const absolutePathResult = createEvaluateResult({
+      provider: { id: `file://${path.join(process.cwd(), 'provider.js')}` },
+      vars: { key: 'value' },
+    });
+
+    expect(resultIsForTestCase(absolutePathResult, relativePathTestCase)).toBe(true);
+  });
+
+  it('matches when test provider has no file:// prefix and result has absolute path', async () => {
+    const noPathTestCase: TestCase = {
+      provider: './provider.js',
+      vars: { key: 'value' },
+    };
+
+    const absolutePathResult = createEvaluateResult({
+      provider: `file://${path.join(process.cwd(), 'provider.js')}` as any,
+      vars: { key: 'value' },
+    });
+
+    expect(resultIsForTestCase(absolutePathResult, noPathTestCase)).toBe(true);
+  });
+
+  it('matches when result.vars has runtime variables like _conversation', async () => {
+    // This tests the fix for issue #5849 - cache behavior regression
+    // result.vars contains runtime variables that should be filtered out
+    const testCase: TestCase = {
+      provider: 'provider',
+      vars: {
+        input: 'hello',
+        language: 'en',
+      },
+    };
+
+    const result = createEvaluateResult({
+      provider: 'provider' as any,
+      vars: {
+        input: 'hello',
+        language: 'en',
+        _conversation: [], // Runtime variable added during evaluation
+      },
+      testCase: {
+        vars: {
+          input: 'hello',
+          language: 'en',
+        },
+      },
+    });
+
+    // Should match because _conversation is filtered out
+    expect(resultIsForTestCase(result, testCase)).toBe(true);
+  });
+
+  it('matches when result.vars has runtime variables and testCase also has them', async () => {
+    // Edge case: both have runtime vars (shouldn't happen in practice but should still work)
+    const testCase: TestCase = {
+      provider: 'provider',
+      vars: {
+        input: 'hello',
+        language: 'en',
+        _conversation: [],
+      },
+    };
+
+    const result = createEvaluateResult({
+      provider: 'provider' as any,
+      vars: {
+        input: 'hello',
+        language: 'en',
+        _conversation: [],
+      },
+    });
+
+    // Should match because both have same vars after filtering
+    expect(resultIsForTestCase(result, testCase)).toBe(true);
+  });
+
+  it('matches when result.vars has sessionId runtime variable from GOAT/Crescendo providers', async () => {
+    // This tests the fix for multi-turn strategy providers (GOAT, Crescendo)
+    // that add sessionId to vars during execution
+    const testCase: TestCase = {
+      provider: 'provider',
+      vars: {
+        prompt: 'test prompt',
+        goal: 'test goal',
+      },
+    };
+
+    const result = createEvaluateResult({
+      provider: 'provider' as any,
+      vars: {
+        prompt: 'test prompt',
+        goal: 'test goal',
+        sessionId: 'goat-session-abc123', // Added by GOAT provider during evaluation
+      },
+    });
+
+    // Should match because sessionId is filtered out
+    expect(resultIsForTestCase(result, testCase)).toBe(true);
+  });
+
+  it('matches when result.vars has both _conversation and sessionId runtime variables', async () => {
+    const testCase: TestCase = {
+      provider: 'provider',
+      vars: {
+        input: 'hello',
+      },
+    };
+
+    const result = createEvaluateResult({
+      provider: 'provider' as any,
+      vars: {
+        input: 'hello',
+        _conversation: [{ role: 'user', content: 'hi' }],
+        sessionId: 'session-xyz789',
+      },
+    });
+
+    // Should match because both runtime vars are filtered out
+    expect(resultIsForTestCase(result, testCase)).toBe(true);
+  });
+
+  it('matches when testCase also has sessionId (both filtered)', async () => {
+    // Edge case: if user explicitly sets sessionId in test config, it should still match
+    const testCase: TestCase = {
+      provider: 'provider',
+      vars: {
+        prompt: 'test',
+        sessionId: 'user-defined-session',
+      },
+    };
+
+    const result = createEvaluateResult({
+      provider: 'provider' as any,
+      vars: {
+        prompt: 'test',
+        sessionId: 'different-runtime-session',
+      },
+    });
+
+    // Should match because sessionId is filtered from both sides
+    expect(resultIsForTestCase(result, testCase)).toBe(true);
+  });
+});
+
+describe('isRuntimeVar', () => {
+  it('should return true for underscore-prefixed variables', () => {
+    expect(isRuntimeVar('_conversation')).toBe(true);
+    expect(isRuntimeVar('_metadata')).toBe(true);
+    expect(isRuntimeVar('_internal')).toBe(true);
+    expect(isRuntimeVar('_')).toBe(true);
+  });
+
+  it('should return true for explicit runtime vars like sessionId', () => {
+    expect(isRuntimeVar('sessionId')).toBe(true);
+  });
+
+  it('should return false for regular variables', () => {
+    expect(isRuntimeVar('prompt')).toBe(false);
+    expect(isRuntimeVar('input')).toBe(false);
+    expect(isRuntimeVar('context')).toBe(false);
+    expect(isRuntimeVar('goal')).toBe(false);
+  });
+
+  it('should return false for variables that contain underscore but do not start with it', () => {
+    expect(isRuntimeVar('my_variable')).toBe(false);
+    expect(isRuntimeVar('some_conversation')).toBe(false);
+    expect(isRuntimeVar('session_id')).toBe(false);
+  });
+});
+
+describe('filterRuntimeVars', () => {
+  it('should filter out _conversation', () => {
+    const vars = { input: 'hello', _conversation: [{ role: 'user', content: 'hi' }] };
+    const result = filterRuntimeVars(vars);
+    expect(result).toEqual({ input: 'hello' });
+    expect(result).not.toHaveProperty('_conversation');
+  });
+
+  it('should filter out sessionId', () => {
+    const vars = { input: 'hello', sessionId: 'test-session-123' };
+    const result = filterRuntimeVars(vars);
+    expect(result).toEqual({ input: 'hello' });
+    expect(result).not.toHaveProperty('sessionId');
+  });
+
+  it('should filter out any underscore-prefixed variables', () => {
+    const vars = {
+      input: 'hello',
+      _conversation: [],
+      _metadata: { key: 'value' },
+      _internal: 'data',
+      _customRuntimeVar: 'test',
+    };
+    const result = filterRuntimeVars(vars);
+    expect(result).toEqual({ input: 'hello' });
+    expect(result).not.toHaveProperty('_conversation');
+    expect(result).not.toHaveProperty('_metadata');
+    expect(result).not.toHaveProperty('_internal');
+    expect(result).not.toHaveProperty('_customRuntimeVar');
+  });
+
+  it('should filter out multiple runtime vars', () => {
+    const vars = {
+      input: 'hello',
+      goal: 'test goal',
+      _conversation: [],
+      sessionId: 'test-session-123',
+    };
+    const result = filterRuntimeVars(vars);
+    expect(result).toEqual({ input: 'hello', goal: 'test goal' });
+    expect(result).not.toHaveProperty('_conversation');
+    expect(result).not.toHaveProperty('sessionId');
+  });
+
+  it('should handle undefined vars', () => {
+    expect(filterRuntimeVars(undefined)).toBeUndefined();
+  });
+
+  it('should handle empty vars', () => {
+    expect(filterRuntimeVars({})).toEqual({});
+  });
+
+  it('should not mutate original vars', () => {
+    const vars = { input: 'hello', sessionId: 'test-123', _conversation: [] };
+    const original = { ...vars };
+    filterRuntimeVars(vars);
+    expect(vars).toEqual(original);
+  });
+
+  it('should preserve all non-runtime vars', () => {
+    const vars = {
+      input: 'hello',
+      output: 'world',
+      context: 'some context',
+      nested: { key: 'value' },
+      sessionId: 'to-be-removed',
+    };
+    const result = filterRuntimeVars(vars);
+    expect(result).toEqual({
+      input: 'hello',
+      output: 'world',
+      context: 'some context',
+      nested: { key: 'value' },
+    });
+  });
+
+  it('should preserve variables with underscore in middle of name', () => {
+    const vars = {
+      my_variable: 'value1',
+      some_conversation: 'value2',
+      session_id: 'value3',
+      _runtime: 'filtered',
+    };
+    const result = filterRuntimeVars(vars);
+    expect(result).toEqual({
+      my_variable: 'value1',
+      some_conversation: 'value2',
+      session_id: 'value3',
+    });
+  });
+});
+
+describe('extractRuntimeVars', () => {
+  it('should extract _conversation', () => {
+    const vars = { input: 'hello', _conversation: [{ role: 'user', content: 'hi' }] };
+    const result = extractRuntimeVars(vars);
+    expect(result).toEqual({ _conversation: [{ role: 'user', content: 'hi' }] });
+  });
+
+  it('should extract sessionId', () => {
+    const vars = { input: 'hello', sessionId: 'test-session-123' };
+    const result = extractRuntimeVars(vars);
+    expect(result).toEqual({ sessionId: 'test-session-123' });
+  });
+
+  it('should extract any underscore-prefixed variables', () => {
+    const vars = {
+      input: 'hello',
+      _conversation: [],
+      _metadata: { key: 'value' },
+      _internal: 'data',
+      _customRuntimeVar: 'test',
+    };
+    const result = extractRuntimeVars(vars);
+    expect(result).toEqual({
+      _conversation: [],
+      _metadata: { key: 'value' },
+      _internal: 'data',
+      _customRuntimeVar: 'test',
+    });
+  });
+
+  it('should extract multiple runtime vars', () => {
+    const vars = {
+      input: 'hello',
+      goal: 'test goal',
+      _conversation: [{ role: 'assistant', content: 'response' }],
+      sessionId: 'test-session-123',
+    };
+    const result = extractRuntimeVars(vars);
+    expect(result).toEqual({
+      _conversation: [{ role: 'assistant', content: 'response' }],
+      sessionId: 'test-session-123',
+    });
+  });
+
+  it('should return undefined for undefined vars', () => {
+    expect(extractRuntimeVars(undefined)).toBeUndefined();
+  });
+
+  it('should return undefined for empty vars', () => {
+    expect(extractRuntimeVars({})).toBeUndefined();
+  });
+
+  it('should return undefined when no runtime vars exist', () => {
+    const vars = { input: 'hello', output: 'world' };
+    expect(extractRuntimeVars(vars)).toBeUndefined();
+  });
+
+  it('should not mutate original vars', () => {
+    const vars = { input: 'hello', sessionId: 'test-123', _conversation: [] };
+    const original = { ...vars };
+    extractRuntimeVars(vars);
+    expect(vars).toEqual(original);
+  });
+
+  it('should not extract variables with underscore in middle of name', () => {
+    const vars = {
+      my_variable: 'value1',
+      some_conversation: 'value2',
+      session_id: 'value3',
+      _runtime: 'filtered',
+    };
+    const result = extractRuntimeVars(vars);
+    expect(result).toEqual({ _runtime: 'filtered' });
+    expect(result).not.toHaveProperty('my_variable');
+    expect(result).not.toHaveProperty('some_conversation');
+    expect(result).not.toHaveProperty('session_id');
+  });
+
+  it('should be inverse of filterRuntimeVars', () => {
+    const vars = {
+      input: 'hello',
+      goal: 'test',
+      _conversation: [{ role: 'user', content: 'hi' }],
+      sessionId: 'session-123',
+      _metadata: { custom: true },
+    };
+    const filtered = filterRuntimeVars(vars);
+    const extracted = extractRuntimeVars(vars);
+
+    // Combining filtered and extracted should give back original vars
+    expect({ ...filtered, ...extracted }).toEqual(vars);
+
+    // No overlap between filtered and extracted
+    const filteredKeys = Object.keys(filtered || {});
+    const extractedKeys = Object.keys(extracted || {});
+    expect(filteredKeys.filter((k) => extractedKeys.includes(k))).toEqual([]);
+  });
+});
+
+describe('getTestCaseDeduplicationKey', () => {
+  it('should generate key from vars and strategyId', () => {
+    const testCase: TestCase = {
+      vars: { prompt: 'hello' },
+      metadata: { strategyId: 'jailbreak' },
+    };
+    const key = getTestCaseDeduplicationKey(testCase);
+    expect(JSON.parse(key)).toEqual({
+      vars: { prompt: 'hello' },
+      strategyId: 'jailbreak',
+    });
+  });
+
+  it('should use "none" for tests without strategyId', () => {
+    const testCase: TestCase = {
+      vars: { prompt: 'hello' },
+    };
+    const key = getTestCaseDeduplicationKey(testCase);
+    expect(JSON.parse(key)).toEqual({
+      vars: { prompt: 'hello' },
+      strategyId: 'none',
+    });
+  });
+
+  it('should filter out runtime vars from key', () => {
+    const testCase: TestCase = {
+      vars: { prompt: 'hello', _conversation: [], sessionId: 'abc' },
+      metadata: { strategyId: 'basic' },
+    };
+    const key = getTestCaseDeduplicationKey(testCase);
+    expect(JSON.parse(key)).toEqual({
+      vars: { prompt: 'hello' },
+      strategyId: 'basic',
+    });
+  });
+
+  it('should handle undefined vars', () => {
+    const testCase: TestCase = {
+      metadata: { strategyId: 'test' },
+    };
+    const key = getTestCaseDeduplicationKey(testCase);
+    expect(JSON.parse(key)).toEqual({
+      vars: undefined,
+      strategyId: 'test',
+    });
+  });
+});
+
+describe('deduplicateTestCases', () => {
+  it('should remove duplicate tests with same vars and strategyId', () => {
+    const tests: TestCase[] = [
+      { vars: { prompt: 'hello' }, metadata: { strategyId: 'basic' } },
+      { vars: { prompt: 'hello' }, metadata: { strategyId: 'basic' } },
+      { vars: { prompt: 'world' }, metadata: { strategyId: 'basic' } },
+    ];
+    const result = deduplicateTestCases(tests);
+    expect(result).toHaveLength(2);
+    expect(result[0].vars).toEqual({ prompt: 'hello' });
+    expect(result[1].vars).toEqual({ prompt: 'world' });
+  });
+
+  it('should keep tests with same vars but different strategyId', () => {
+    const tests: TestCase[] = [
+      { vars: { prompt: 'hello' }, metadata: { strategyId: 'basic' } },
+      { vars: { prompt: 'hello' }, metadata: { strategyId: 'jailbreak' } },
+    ];
+    const result = deduplicateTestCases(tests);
+    expect(result).toHaveLength(2);
+  });
+
+  it('should filter runtime vars when checking for duplicates', () => {
+    const tests: TestCase[] = [
+      { vars: { prompt: 'hello', sessionId: 'abc' }, metadata: { strategyId: 'basic' } },
+      { vars: { prompt: 'hello', sessionId: 'xyz' }, metadata: { strategyId: 'basic' } },
+    ];
+    const result = deduplicateTestCases(tests);
+    // These should be considered duplicates after filtering sessionId
+    expect(result).toHaveLength(1);
+  });
+
+  it('should handle empty array', () => {
+    expect(deduplicateTestCases([])).toEqual([]);
+  });
+
+  it('should preserve order (keep first occurrence)', () => {
+    const tests: TestCase[] = [
+      { vars: { prompt: 'first' }, metadata: { strategyId: 'basic', order: 1 } },
+      { vars: { prompt: 'first' }, metadata: { strategyId: 'basic', order: 2 } },
+    ];
+    const result = deduplicateTestCases(tests);
+    expect(result).toHaveLength(1);
+    expect(result[0].metadata?.order).toBe(1);
+  });
+});

test/util/config/default.test.ts

@@ -0,0 +1,177 @@
+import path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { configCache, loadDefaultConfig } from '../../../src/util/config/default';
+import { maybeReadConfig } from '../../../src/util/config/load';
+
+vi.mock('../../../src/util/config/load', () => ({
+  maybeReadConfig: vi.fn(),
+}));
+
+describe('loadDefaultConfig', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.spyOn(process, 'cwd').mockImplementation(() => '/test/path');
+    configCache.clear();
+  });
+
+  it('should return empty config when no config file is found', async () => {
+    vi.mocked(maybeReadConfig).mockResolvedValue(undefined);
+
+    const result = await loadDefaultConfig();
+    expect(result).toEqual({
+      defaultConfig: {},
+      defaultConfigPath: undefined,
+    });
+    expect(maybeReadConfig).toHaveBeenCalledTimes(9);
+    expect(maybeReadConfig).toHaveBeenNthCalledWith(
+      1,
+      path.normalize('/test/path/promptfooconfig.yaml'),
+    );
+  });
+
+  it('should return the first valid config file found', async () => {
+    const mockConfig = { prompts: ['Some prompt'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig)
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce(mockConfig);
+
+    const result = await loadDefaultConfig();
+    expect(result).toEqual({
+      defaultConfig: mockConfig,
+      defaultConfigPath: path.normalize('/test/path/promptfooconfig.json'),
+    });
+    expect(maybeReadConfig).toHaveBeenCalledTimes(3);
+  });
+
+  it('should stop checking extensions after finding a valid config', async () => {
+    const mockConfig = { prompts: ['Some prompt'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(undefined).mockResolvedValueOnce(mockConfig);
+
+    await loadDefaultConfig();
+
+    expect(maybeReadConfig).toHaveBeenCalledTimes(2);
+    expect(maybeReadConfig).toHaveBeenNthCalledWith(
+      1,
+      path.normalize('/test/path/promptfooconfig.yaml'),
+    );
+    expect(maybeReadConfig).toHaveBeenNthCalledWith(
+      2,
+      path.normalize('/test/path/promptfooconfig.yml'),
+    );
+  });
+
+  it('should use provided directory when specified', async () => {
+    const mockConfig = { prompts: ['Some prompt'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(mockConfig);
+
+    const customDir = '/custom/directory';
+    const result = await loadDefaultConfig(customDir);
+    expect(result).toEqual({
+      defaultConfig: mockConfig,
+      defaultConfigPath: path.join(customDir, 'promptfooconfig.yaml'),
+    });
+    expect(maybeReadConfig).toHaveBeenCalledWith(path.join(customDir, 'promptfooconfig.yaml'));
+  });
+
+  it('should use custom config name when provided', async () => {
+    const mockConfig = { prompts: ['Custom config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(mockConfig);
+
+    const result = await loadDefaultConfig(undefined, 'redteam');
+    expect(result).toEqual({
+      defaultConfig: mockConfig,
+      defaultConfigPath: path.normalize('/test/path/redteam.yaml'),
+    });
+    expect(maybeReadConfig).toHaveBeenCalledWith(path.normalize('/test/path/redteam.yaml'));
+  });
+
+  it('should use different caches for different config names', async () => {
+    const mockConfig1 = { prompts: ['Config 1'], providers: [], tests: [] };
+    const mockConfig2 = { prompts: ['Config 2'], providers: [], tests: [] };
+
+    vi.mocked(maybeReadConfig)
+      .mockResolvedValueOnce(mockConfig1)
+      .mockResolvedValueOnce(mockConfig2);
+
+    const result1 = await loadDefaultConfig(undefined, 'promptfooconfig');
+    const result2 = await loadDefaultConfig(undefined, 'redteam');
+
+    expect(result1).not.toEqual(result2);
+    expect(result1.defaultConfig).toEqual(mockConfig1);
+    expect(result2.defaultConfig).toEqual(mockConfig2);
+
+    const cachedResult1 = await loadDefaultConfig(undefined, 'promptfooconfig');
+    const cachedResult2 = await loadDefaultConfig(undefined, 'redteam');
+
+    expect(cachedResult1).toEqual(result1);
+    expect(cachedResult2).toEqual(result2);
+    expect(maybeReadConfig).toHaveBeenCalledTimes(2);
+  });
+
+  it('should use different caches for different directories', async () => {
+    const mockConfig1 = { prompts: ['Config 1'], providers: [], tests: [] };
+    const mockConfig2 = { prompts: ['Config 2'], providers: [], tests: [] };
+
+    vi.mocked(maybeReadConfig)
+      .mockResolvedValueOnce(mockConfig1)
+      .mockResolvedValueOnce(mockConfig2);
+
+    const dir1 = '/dir1';
+    const dir2 = '/dir2';
+
+    const result1 = await loadDefaultConfig(dir1);
+    const result2 = await loadDefaultConfig(dir2);
+
+    expect(result1).not.toEqual(result2);
+    expect(result1.defaultConfig).toEqual(mockConfig1);
+    expect(result2.defaultConfig).toEqual(mockConfig2);
+
+    const cachedResult1 = await loadDefaultConfig(dir1);
+    const cachedResult2 = await loadDefaultConfig(dir2);
+
+    expect(cachedResult1).toEqual(result1);
+    expect(cachedResult2).toEqual(result2);
+    expect(maybeReadConfig).toHaveBeenCalledTimes(2);
+  });
+
+  it('should use cache for subsequent calls with same parameters', async () => {
+    const mockConfig = { prompts: ['Cached config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(mockConfig);
+
+    const result1 = await loadDefaultConfig();
+    const result2 = await loadDefaultConfig();
+
+    expect(result1).toEqual(result2);
+    expect(maybeReadConfig).toHaveBeenCalledTimes(1);
+  });
+
+  it('should handle errors when reading config files', async () => {
+    vi.mocked(maybeReadConfig).mockRejectedValue(new Error('Permission denied'));
+
+    await expect(loadDefaultConfig()).rejects.toThrow('Permission denied');
+  });
+
+  it('should handle various config names', async () => {
+    const mockConfig = { prompts: ['Test config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValue(mockConfig);
+
+    const configNames = ['test1', 'test2', 'test3'];
+    for (const name of configNames) {
+      const result = await loadDefaultConfig(undefined, name);
+      expect(result.defaultConfigPath).toContain(name);
+    }
+  });
+
+  it('should handle interaction between configName and directory', async () => {
+    const mockConfig = { prompts: ['Combined config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValue(mockConfig);
+
+    const customDir = '/custom/dir';
+    const customName = 'customconfig';
+    const result = await loadDefaultConfig(customDir, customName);
+
+    expect(result.defaultConfigPath).toEqual(path.join(customDir, `${customName}.yaml`));
+  });
+});

test/util/config/main.test.ts

@@ -0,0 +1,244 @@
+import * as os from 'os';
+import * as path from 'path';
+
+import * as yaml from 'js-yaml';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import cliState from '../../../src/cliState';
+import logger from '../../../src/logger';
+import {
+  getConfigDirectoryPath,
+  refreshConfigDirectoryPathFromEnv,
+  setConfigDirectoryPath,
+} from '../../../src/util/config/manage';
+import { writePromptfooConfig } from '../../../src/util/config/writer';
+import { mockProcessEnv } from '../../util/utils';
+
+import type { UnifiedConfig } from '../../../src/types/index';
+
+// Create hoisted mock functions for fs
+const mockFs = vi.hoisted(() => ({
+  existsSync: vi.fn(),
+  mkdirSync: vi.fn(),
+  writeFileSync: vi.fn(),
+  readFileSync: vi.fn(),
+  readdirSync: vi.fn(),
+  statSync: vi.fn(),
+  unlinkSync: vi.fn(),
+  rmdirSync: vi.fn(),
+  copyFileSync: vi.fn(),
+  renameSync: vi.fn(),
+  accessSync: vi.fn(),
+  chmodSync: vi.fn(),
+  constants: {},
+}));
+
+// Mock both 'fs' and 'node:fs' to cover all import patterns
+vi.mock('fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+vi.mock('node:fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+vi.mock('os');
+// js-yaml v5 is native ESM with a sealed namespace, so vi.spyOn cannot patch it.
+// Wrap dump in a spy-able mock that keeps the real implementation.
+vi.mock('js-yaml', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('js-yaml')>();
+  return { ...actual, dump: vi.fn(actual.dump) };
+});
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+describe('config', () => {
+  const mockHomedir = '/mock/home';
+  const defaultConfigPath = path.join(mockHomedir, '.promptfoo');
+  let restoreEnv: () => void;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(os.homedir).mockReturnValue(mockHomedir);
+    mockFs.existsSync.mockReturnValue(false);
+    restoreEnv = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: undefined });
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  afterEach(() => {
+    restoreEnv();
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  describe('getConfigDirectoryPath', () => {
+    it('returns default path when no custom path is set', () => {
+      expect(getConfigDirectoryPath()).toBe(defaultConfigPath);
+    });
+
+    it('reads a config directory refreshed after early env-file loading', () => {
+      const restoreConfigDir = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: '/env-file/config' });
+      try {
+        refreshConfigDirectoryPathFromEnv();
+        expect(getConfigDirectoryPath()).toBe('/env-file/config');
+      } finally {
+        restoreConfigDir();
+        refreshConfigDirectoryPathFromEnv();
+      }
+    });
+
+    it('does not let eval config env overrides move the global config directory', () => {
+      const originalConfig = cliState.config;
+      cliState.config = { env: { PROMPTFOO_CONFIG_DIR: '/eval-config/path' } };
+      try {
+        expect(getConfigDirectoryPath()).toBe(defaultConfigPath);
+      } finally {
+        cliState.config = originalConfig;
+      }
+    });
+
+    it('does not move after later process environment changes', () => {
+      const restoreConfigDir = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: '/late/config' });
+      try {
+        expect(getConfigDirectoryPath()).toBe(defaultConfigPath);
+      } finally {
+        restoreConfigDir();
+      }
+    });
+
+    it('does not create directory when createIfNotExists is false', () => {
+      getConfigDirectoryPath(false);
+      expect(mockFs.mkdirSync).not.toHaveBeenCalled();
+    });
+
+    // Note: The directory creation test cannot verify mkdirSync was called because
+    // the source uses require('fs') inside the function which bypasses Vitest's ESM mocking.
+    // We verify the function completes successfully with the correct return value instead.
+    it('handles createIfNotExists flag and returns correct path', () => {
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(defaultConfigPath);
+    });
+
+    it('does not create directory when it already exists', () => {
+      mockFs.existsSync.mockReturnValue(true);
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(defaultConfigPath);
+    });
+  });
+
+  describe('setConfigDirectoryPath', () => {
+    it('updates the config directory path', () => {
+      const newPath = '/new/config/path';
+      setConfigDirectoryPath(newPath);
+      expect(getConfigDirectoryPath()).toBe(newPath);
+    });
+
+    it('overrides the environment variable', () => {
+      const envPath = '/env/path';
+      const newPath = '/new/path';
+      const restoreConfigDir = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: envPath });
+      try {
+        setConfigDirectoryPath(newPath);
+        expect(getConfigDirectoryPath()).toBe(newPath);
+      } finally {
+        restoreConfigDir();
+      }
+    });
+  });
+});
+
+describe('writePromptfooConfig', () => {
+  const mockOutputPath = '/mock/output/path.yaml';
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('writes a basic config to the specified path', () => {
+    const mockConfig: Partial<UnifiedConfig> = { description: 'Test config' };
+    const mockYaml = 'description: Test config\n';
+    vi.mocked(yaml.dump).mockReturnValue(mockYaml);
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    expect(mockFs.writeFileSync).toHaveBeenCalledWith(
+      mockOutputPath,
+      `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n${mockYaml}`,
+    );
+  });
+
+  it('orders the keys of the config correctly', () => {
+    const mockConfig: Partial<UnifiedConfig> = {
+      tests: [{ assert: [{ type: 'equals', value: 'test assertion' }] }],
+      description: 'Test config',
+      prompts: ['prompt1'],
+      providers: ['provider1'],
+      defaultTest: { assert: [{ type: 'equals', value: 'default assertion' }] },
+    };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    const dumpCall = vi.mocked(yaml.dump).mock.calls[0][0];
+    const keys = Object.keys(dumpCall);
+    expect(keys).toEqual(['description', 'prompts', 'providers', 'defaultTest', 'tests']);
+  });
+
+  it('uses js-yaml to dump the config with skipInvalid option', () => {
+    const mockConfig: Partial<UnifiedConfig> = { description: 'Test config' };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    expect(yaml.dump).toHaveBeenCalledWith(expect.anything(), { skipInvalid: true });
+  });
+
+  it('handles empty config', () => {
+    vi.mocked(yaml.dump).mockReturnValueOnce('');
+    const mockConfig: Partial<UnifiedConfig> = {};
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+    expect(mockFs.writeFileSync).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalledWith('Warning: config is empty, skipping write');
+  });
+
+  it('preserves all fields of the UnifiedConfig', () => {
+    const mockConfig: Partial<UnifiedConfig> = {
+      description: 'Full config test',
+      prompts: ['prompt1', 'prompt2'],
+      providers: ['provider1', 'provider2'],
+      defaultTest: { assert: [{ type: 'equals', value: 'default assertion' }] },
+      tests: [
+        { assert: [{ type: 'equals', value: 'test assertion 1' }] },
+        { assert: [{ type: 'equals', value: 'test assertion 2' }] },
+      ],
+      outputPath: './output',
+    };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    const dumpCall = vi.mocked(yaml.dump).mock.calls[0][0];
+    expect(dumpCall).toEqual(expect.objectContaining(mockConfig));
+  });
+
+  it('handles config with undefined values', () => {
+    const mockConfig: Partial<UnifiedConfig> = {
+      description: 'Config with undefined',
+      prompts: undefined,
+      providers: ['provider1'],
+    };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    const dumpCall = vi.mocked(yaml.dump).mock.calls[0][0];
+    expect(dumpCall).toHaveProperty('description', 'Config with undefined');
+    expect(dumpCall).toHaveProperty('providers', ['provider1']);
+    expect(dumpCall).not.toHaveProperty('prompts');
+  });
+});

test/util/config/manage.test.ts

@@ -0,0 +1,173 @@
+import * as os from 'os';
+import * as path from 'path';
+
+import * as yaml from 'js-yaml';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+  getConfigDirectoryPath,
+  refreshConfigDirectoryPathFromEnv,
+  setConfigDirectoryPath,
+} from '../../../src/util/config/manage';
+import { writePromptfooConfig } from '../../../src/util/config/writer';
+import { mockProcessEnv } from '../../util/utils';
+
+// Create hoisted mock functions for fs
+const mockFs = vi.hoisted(() => ({
+  existsSync: vi.fn(),
+  mkdirSync: vi.fn(),
+  writeFileSync: vi.fn(),
+  readFileSync: vi.fn(),
+  readdirSync: vi.fn(),
+  statSync: vi.fn(),
+  unlinkSync: vi.fn(),
+  rmdirSync: vi.fn(),
+  copyFileSync: vi.fn(),
+  renameSync: vi.fn(),
+  accessSync: vi.fn(),
+  chmodSync: vi.fn(),
+  constants: {},
+}));
+
+// Mock both 'fs' and 'node:fs' to cover all import patterns
+vi.mock('fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+vi.mock('node:fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+// js-yaml v5 is native ESM with a sealed namespace, so vi.spyOn cannot patch it.
+// Wrap dump in a spy-able mock that keeps the real implementation.
+vi.mock('js-yaml', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('js-yaml')>();
+  return { ...actual, dump: vi.fn(actual.dump) };
+});
+
+// Mock os module
+vi.mock('os');
+vi.mock('../../../src/logger', () => ({
+  default: {
+    warn: vi.fn(),
+  },
+}));
+
+describe('config management', () => {
+  let restoreEnv: () => void;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(os.homedir).mockReturnValue('/home/user');
+    restoreEnv = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: undefined });
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  afterEach(() => {
+    restoreEnv();
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  describe('getConfigDirectoryPath', () => {
+    it('should return default config path when no custom path set', () => {
+      setConfigDirectoryPath(undefined);
+      const configPath = getConfigDirectoryPath();
+      expect(configPath).toBe(path.join('/home/user', '.promptfoo'));
+    });
+
+    // Note: The directory creation test cannot verify mkdirSync was called because
+    // the source uses require('fs') inside the function which bypasses Vitest's ESM mocking.
+    // We verify the function completes successfully with the correct return value instead.
+    it('should handle createIfNotExists flag and return correct path', () => {
+      mockFs.existsSync.mockReturnValue(false);
+      setConfigDirectoryPath(undefined);
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(path.join('/home/user', '.promptfoo'));
+    });
+
+    it('should not create directory if it already exists', () => {
+      mockFs.existsSync.mockReturnValue(true);
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(path.join('/home/user', '.promptfoo'));
+    });
+  });
+
+  describe('setConfigDirectoryPath', () => {
+    it('should set custom config directory path', () => {
+      const customPath = '/custom/path';
+      setConfigDirectoryPath(customPath);
+      const configPath = getConfigDirectoryPath();
+      expect(configPath).toBe(customPath);
+    });
+
+    it('should handle undefined path', () => {
+      setConfigDirectoryPath(undefined);
+      const configPath = getConfigDirectoryPath();
+      expect(configPath).toBe(path.join('/home/user', '.promptfoo'));
+    });
+  });
+
+  describe('writePromptfooConfig', () => {
+    const outputPath = 'config.yaml';
+
+    it('should write config with schema comment', () => {
+      const config = {
+        description: 'test config',
+        prompts: ['prompt1'],
+      };
+
+      writePromptfooConfig(config, outputPath);
+
+      expect(mockFs.writeFileSync).toHaveBeenCalledWith(
+        outputPath,
+        `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n${yaml.dump(config)}`,
+      );
+    });
+
+    it('should write config with header comments', () => {
+      const config = {
+        description: 'test config',
+      };
+      const headerComments = ['Comment 1', 'Comment 2'];
+
+      writePromptfooConfig(config, outputPath, headerComments);
+
+      const expectedContent =
+        `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n` +
+        `# Comment 1\n# Comment 2\n${yaml.dump(config)}`;
+
+      expect(mockFs.writeFileSync).toHaveBeenCalledWith(outputPath, expectedContent);
+    });
+
+    it('should handle empty config', () => {
+      const config = {};
+      const result = writePromptfooConfig(config, outputPath);
+      expect(result).toEqual({});
+    });
+
+    it('should order config keys', () => {
+      const config = {
+        tests: ['test1'],
+        description: 'desc',
+        prompts: ['prompt1'],
+      };
+
+      const result = writePromptfooConfig(config, outputPath);
+
+      expect(Object.keys(result)[0]).toBe('description');
+      expect(Object.keys(result)[1]).toBe('prompts');
+      expect(Object.keys(result)[2]).toBe('tests');
+    });
+
+    it('should handle empty yaml content', () => {
+      vi.mocked(yaml.dump).mockReturnValueOnce('');
+      const config = { description: 'test' };
+      const result = writePromptfooConfig(config, outputPath);
+      expect(result).toEqual(config);
+      expect(mockFs.writeFileSync).not.toHaveBeenCalled();
+    });
+  });
+});

test/util/config/scenarios.test.ts

@@ -0,0 +1,239 @@
+import * as fs from 'fs';
+import * as fsPromises from 'fs/promises';
+
+import { globSync } from 'glob';
+import * as yaml from 'js-yaml';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { validateAssertions } from '../../../src/assertions/validateAssertions';
+import cliState from '../../../src/cliState';
+import { readPrompts, readProviderPromptMap } from '../../../src/prompts/index';
+import { loadApiProviders } from '../../../src/providers/index';
+import { type TestCase } from '../../../src/types/index';
+// Import after mocking
+import { resolveConfigs } from '../../../src/util/config/load';
+import { maybeLoadFromExternalFile } from '../../../src/util/file';
+import { readFilters } from '../../../src/util/index';
+import { readTests } from '../../../src/util/testCaseReader';
+import { createMockProvider } from '../../factories/provider';
+
+vi.mock('fs');
+vi.mock('fs/promises');
+vi.mock('glob', () => ({
+  globSync: vi.fn(),
+  hasMagic: vi.fn((pattern: string | string[]) => {
+    const p = Array.isArray(pattern) ? pattern.join('') : pattern;
+    return p.includes('*') || p.includes('?') || p.includes('[') || p.includes('{');
+  }),
+}));
+
+// Mock all the dependencies first
+vi.mock('../../../src/util/file', async () => {
+  const actual =
+    await vi.importActual<typeof import('../../../src/util/file')>('../../../src/util/file');
+  return {
+    ...actual,
+    maybeLoadFromExternalFile: vi.fn(),
+  };
+});
+vi.mock('../../../src/prompts');
+vi.mock('../../../src/providers');
+vi.mock('../../../src/util/testCaseReader');
+vi.mock('../../../src/util', async () => {
+  const actual = await vi.importActual<typeof import('../../../src/util')>('../../../src/util');
+  return {
+    ...actual,
+    readFilters: vi.fn(),
+  };
+});
+vi.mock('../../../src/assertions/validateAssertions');
+
+describe('Scenario loading with glob patterns', () => {
+  const originalBasePath = cliState.basePath;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    cliState.basePath = '/test/path';
+
+    // Setup default mocks
+    vi.mocked(readPrompts).mockResolvedValue([{ raw: 'Test prompt', label: 'Test prompt' }]);
+    vi.mocked(readProviderPromptMap).mockReturnValue({});
+    vi.mocked(loadApiProviders).mockResolvedValue([
+      createMockProvider({ id: 'openai:gpt-3.5-turbo' }),
+    ]);
+    vi.mocked(readTests).mockImplementation(async (tests) =>
+      Array.isArray(tests) ? (tests as TestCase[]) : [],
+    );
+    vi.mocked(readFilters).mockResolvedValue({});
+    vi.mocked(validateAssertions).mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    cliState.basePath = originalBasePath;
+  });
+
+  it('should flatten scenarios when loaded with glob patterns', async () => {
+    const scenario1 = {
+      description: 'Scenario 1',
+      config: [{ vars: { name: 'Alice' } }],
+      tests: [{ vars: { question: 'Test 1' } }],
+    };
+
+    const scenario2 = {
+      description: 'Scenario 2',
+      config: [{ vars: { name: 'Bob' } }],
+      tests: [{ vars: { question: 'Test 2' } }],
+    };
+
+    // Mock file system
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fsPromises.readFile).mockImplementation((filePath) => {
+      if (filePath === 'config.yaml') {
+        return Promise.resolve(
+          yaml.dump({
+            prompts: ['Test prompt'],
+            providers: ['openai:gpt-3.5-turbo'],
+            scenarios: ['file://scenarios/*.yaml'],
+          }),
+        );
+      }
+      return Promise.resolve('');
+    });
+
+    // Mock glob to return config file
+    vi.mocked(globSync).mockReturnValue(['config.yaml']);
+
+    // Mock maybeLoadFromExternalFile to return nested array (simulating glob expansion)
+    vi.mocked(maybeLoadFromExternalFile).mockImplementation((input) => {
+      if (Array.isArray(input) && input[0] === 'file://scenarios/*.yaml') {
+        // Return nested array as would happen with glob pattern
+        return [[scenario1, scenario2]];
+      }
+      return input;
+    });
+
+    const cmdObj = { config: ['config.yaml'] };
+    const defaultConfig = {};
+
+    const { testSuite } = await resolveConfigs(cmdObj, defaultConfig);
+
+    // Check if maybeLoadFromExternalFile was called with the expected argument
+    expect(maybeLoadFromExternalFile).toHaveBeenCalledWith(['file://scenarios/*.yaml']);
+
+    // Verify scenarios are flattened correctly
+    expect(testSuite.scenarios).toHaveLength(2);
+    expect(testSuite.scenarios![0]).toEqual(scenario1);
+    expect(testSuite.scenarios![1]).toEqual(scenario2);
+  });
+
+  it('should handle multiple scenario files with glob patterns', async () => {
+    const scenarios = [
+      {
+        description: 'Scenario A',
+        config: [{ vars: { test: 'A' } }],
+        tests: [{ vars: { input: '1' } }],
+      },
+      {
+        description: 'Scenario B',
+        config: [{ vars: { test: 'B' } }],
+        tests: [{ vars: { input: '2' } }],
+      },
+      {
+        description: 'Scenario C',
+        config: [{ vars: { test: 'C' } }],
+        tests: [{ vars: { input: '3' } }],
+      },
+    ];
+
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fsPromises.readFile).mockImplementation((filePath) => {
+      if (filePath === 'config.yaml') {
+        return Promise.resolve(
+          yaml.dump({
+            prompts: ['Test prompt'],
+            providers: ['openai:gpt-3.5-turbo'],
+            scenarios: ['file://group1/*.yaml', 'file://group2/*.yaml'],
+          }),
+        );
+      }
+      return Promise.resolve('');
+    });
+
+    vi.mocked(globSync).mockReturnValue(['config.yaml']);
+
+    vi.mocked(maybeLoadFromExternalFile).mockImplementation((input) => {
+      if (Array.isArray(input)) {
+        // Simulate two glob patterns each returning different scenarios
+        return [
+          [scenarios[0], scenarios[1]], // group1/*.yaml
+          [scenarios[2]], // group2/*.yaml
+        ];
+      }
+      return input;
+    });
+
+    const cmdObj = { config: ['config.yaml'] };
+    const defaultConfig = {};
+
+    const { testSuite } = await resolveConfigs(cmdObj, defaultConfig);
+
+    // Verify all scenarios are flattened into a single array
+    expect(testSuite.scenarios).toHaveLength(3);
+    expect(testSuite.scenarios).toEqual(scenarios);
+  });
+
+  it('should handle mixed scenario loading (direct and glob)', async () => {
+    const directScenario = {
+      description: 'Direct scenario',
+      config: [{ vars: { type: 'direct' } }],
+      tests: [{ vars: { test: 'direct' } }],
+    };
+
+    const globScenarios = [
+      {
+        description: 'Glob scenario 1',
+        config: [{ vars: { type: 'glob' } }],
+        tests: [{ vars: { test: 'glob1' } }],
+      },
+      {
+        description: 'Glob scenario 2',
+        config: [{ vars: { type: 'glob' } }],
+        tests: [{ vars: { test: 'glob2' } }],
+      },
+    ];
+
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fsPromises.readFile).mockImplementation((filePath) => {
+      if (filePath === 'config.yaml') {
+        return Promise.resolve(
+          yaml.dump({
+            prompts: ['Test prompt'],
+            providers: ['openai:gpt-3.5-turbo'],
+            scenarios: [directScenario, 'file://scenarios/*.yaml'],
+          }),
+        );
+      }
+      return Promise.resolve('');
+    });
+
+    vi.mocked(globSync).mockReturnValue(['config.yaml']);
+
+    vi.mocked(maybeLoadFromExternalFile).mockImplementation((input) => {
+      if (Array.isArray(input)) {
+        // First element is direct scenario, second is glob pattern
+        return [directScenario, globScenarios];
+      }
+      return input;
+    });
+
+    const cmdObj = { config: ['config.yaml'] };
+    const defaultConfig = {};
+
+    const { testSuite } = await resolveConfigs(cmdObj, defaultConfig);
+
+    // Verify mixed scenarios are flattened correctly
+    expect(testSuite.scenarios).toHaveLength(3);
+    expect(testSuite.scenarios![0]).toEqual(directScenario);
+    expect(testSuite.scenarios![1]).toEqual(globScenarios[0]);
+    expect(testSuite.scenarios![2]).toEqual(globScenarios[1]);
+  });
+});

test/util/dataUrl.test.ts

@@ -0,0 +1,309 @@
+import { describe, expect, it } from 'vitest';
+import {
+  extractBase64FromDataUrl,
+  isDataUrl,
+  parseDataUrl,
+  toDataUri,
+} from '../../src/util/dataUrl';
+
+describe('dataUrl utilities', () => {
+  describe('isDataUrl', () => {
+    it('should return true for valid data URLs', () => {
+      expect(isDataUrl('data:image/jpeg;base64,/9j/4AAQSkZJRg')).toBe(true);
+      expect(isDataUrl('data:image/png;base64,iVBORw0KGgo')).toBe(true);
+      expect(isDataUrl('data:image/gif;base64,R0lGODlh')).toBe(true);
+      expect(isDataUrl('data:image/webp;base64,UklGR')).toBe(true);
+      expect(isDataUrl('data:text/plain;base64,SGVsbG8=')).toBe(true);
+    });
+
+    it('should return false for raw base64 strings', () => {
+      expect(isDataUrl('/9j/4AAQSkZJRg')).toBe(false);
+      expect(isDataUrl('iVBORw0KGgo')).toBe(false);
+      expect(isDataUrl('R0lGODlh')).toBe(false);
+    });
+
+    it('should return false for HTTP URLs', () => {
+      expect(isDataUrl('https://example.com/image.jpg')).toBe(false);
+      expect(isDataUrl('http://example.com/image.png')).toBe(false);
+    });
+
+    it('should return false for file URLs', () => {
+      expect(isDataUrl('file://path/to/image.jpg')).toBe(false);
+    });
+
+    it('should return false for empty or invalid strings', () => {
+      expect(isDataUrl('')).toBe(false);
+      expect(isDataUrl('not a url')).toBe(false);
+      expect(isDataUrl('data')).toBe(false);
+      expect(isDataUrl('data:')).toBe(false);
+    });
+
+    it('should return false for non-string values', () => {
+      expect(isDataUrl(null as any)).toBe(false);
+      expect(isDataUrl(undefined as any)).toBe(false);
+      expect(isDataUrl(123 as any)).toBe(false);
+      expect(isDataUrl({} as any)).toBe(false);
+    });
+  });
+
+  describe('parseDataUrl', () => {
+    it('should parse valid JPEG data URL', () => {
+      const result = parseDataUrl('data:image/jpeg;base64,/9j/4AAQSkZJRg');
+      expect(result).toEqual({
+        mimeType: 'image/jpeg',
+        base64Data: '/9j/4AAQSkZJRg',
+      });
+    });
+
+    it('should parse valid PNG data URL', () => {
+      const result = parseDataUrl('data:image/png;base64,iVBORw0KGgo');
+      expect(result).toEqual({
+        mimeType: 'image/png',
+        base64Data: 'iVBORw0KGgo',
+      });
+    });
+
+    it('should parse valid GIF data URL', () => {
+      const result = parseDataUrl('data:image/gif;base64,R0lGODlh');
+      expect(result).toEqual({
+        mimeType: 'image/gif',
+        base64Data: 'R0lGODlh',
+      });
+    });
+
+    it('should parse valid WebP data URL', () => {
+      const result = parseDataUrl('data:image/webp;base64,UklGR');
+      expect(result).toEqual({
+        mimeType: 'image/webp',
+        base64Data: 'UklGR',
+      });
+    });
+
+    it('should parse data URL with long base64 data', () => {
+      const longBase64 =
+        '/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wgARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUAQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAGgP//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAQUCf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQMBAT8Bf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Bf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEABj8Cf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAT8hf//aAAwDAQACAAMAAAAQ/wD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/EH//xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/EH//xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/EH//2Q==';
+      const result = parseDataUrl(`data:image/jpeg;base64,${longBase64}`);
+      expect(result).toEqual({
+        mimeType: 'image/jpeg',
+        base64Data: longBase64,
+      });
+    });
+
+    it('should return null for raw base64 strings', () => {
+      expect(parseDataUrl('/9j/4AAQSkZJRg')).toBeNull();
+      expect(parseDataUrl('iVBORw0KGgo')).toBeNull();
+    });
+
+    it('should return null for HTTP URLs', () => {
+      expect(parseDataUrl('https://example.com/image.jpg')).toBeNull();
+    });
+
+    it('should return null for malformed data URLs', () => {
+      expect(parseDataUrl('data:image/jpeg')).toBeNull();
+      expect(parseDataUrl('data:image/jpeg;base64')).toBeNull();
+      expect(parseDataUrl('data:image/jpeg;base64,')).toBeNull();
+      expect(parseDataUrl('data:;base64,/9j/')).toBeNull(); // missing mime type
+    });
+
+    it('should return null for data URLs without base64 encoding', () => {
+      expect(parseDataUrl('data:text/plain,Hello World')).toBeNull();
+      expect(parseDataUrl('data:image/jpeg,not-base64')).toBeNull();
+    });
+
+    it('should return null for empty string', () => {
+      expect(parseDataUrl('')).toBeNull();
+    });
+
+    it('should handle data URLs with special characters in MIME type', () => {
+      const result = parseDataUrl('data:image/svg+xml;base64,PHN2Zz4=');
+      expect(result).toEqual({
+        mimeType: 'image/svg+xml',
+        base64Data: 'PHN2Zz4=',
+      });
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle data URLs with charset parameter', () => {
+      const result = parseDataUrl('data:image/jpeg;charset=utf-8;base64,/9j/test');
+      expect(result).toEqual({
+        mimeType: 'image/jpeg',
+        base64Data: '/9j/test',
+      });
+    });
+
+    it('should handle data URLs with multiple parameters', () => {
+      const result = parseDataUrl('data:image/jpeg;name=photo.jpg;charset=utf-8;base64,/9j/test');
+      expect(result).toEqual({
+        mimeType: 'image/jpeg',
+        base64Data: '/9j/test',
+      });
+    });
+
+    it('should trim whitespace from base64 data', () => {
+      const result = parseDataUrl('data:image/jpeg;base64, /9j/test ');
+      expect(result?.base64Data).toBe('/9j/test');
+    });
+
+    it('should trim whitespace from MIME type', () => {
+      const result = parseDataUrl('data: image/jpeg ;base64,/9j/test');
+      expect(result?.mimeType).toBe('image/jpeg');
+    });
+
+    it('should handle empty base64 data gracefully', () => {
+      const result = parseDataUrl('data:image/jpeg;base64,');
+      expect(result).toBeNull();
+    });
+
+    it('should reject case-insensitive data URLs', () => {
+      expect(isDataUrl('DATA:image/jpeg;base64,test')).toBe(false);
+      expect(isDataUrl('Data:image/jpeg;base64,test')).toBe(false);
+      expect(isDataUrl('dAtA:image/jpeg;base64,test')).toBe(false);
+    });
+
+    it('should reject data URLs with newlines in base64', () => {
+      const result = parseDataUrl('data:image/jpeg;base64,/9j/\n4AAQ\ntest');
+      expect(result).toBeNull();
+    });
+
+    it('should handle very short valid base64 (small GIFs)', () => {
+      // Smallest valid 1x1 GIF is ~35 chars, we support down to 20
+      const smallBase64 = 'R0lGODlhAQABAAAAACw='; // 20 chars
+      const result = parseDataUrl(`data:image/gif;base64,${smallBase64}`);
+      expect(result).toEqual({
+        mimeType: 'image/gif',
+        base64Data: smallBase64,
+      });
+    });
+
+    it('should extract base64 from data URLs with charset parameter', () => {
+      const base64 = '/9j/test';
+      const dataUrl = `data:image/jpeg;charset=utf-8;base64,${base64}`;
+      expect(extractBase64FromDataUrl(dataUrl)).toBe(base64);
+    });
+
+    it('should handle URL-encoded SVG (not supported, should fail gracefully)', () => {
+      const urlEncodedSvg = 'data:image/svg+xml,%3Csvg%3E%3C/svg%3E';
+      expect(parseDataUrl(urlEncodedSvg)).toBeNull();
+      // Should pass through unchanged since it's not recognized as data URL
+      expect(extractBase64FromDataUrl(urlEncodedSvg)).toBe(urlEncodedSvg);
+    });
+  });
+
+  describe('extractBase64FromDataUrl', () => {
+    it('should extract base64 from JPEG data URL', () => {
+      const base64 = '/9j/4AAQSkZJRg';
+      const dataUrl = `data:image/jpeg;base64,${base64}`;
+      expect(extractBase64FromDataUrl(dataUrl)).toBe(base64);
+    });
+
+    it('should extract base64 from PNG data URL', () => {
+      const base64 = 'iVBORw0KGgo';
+      const dataUrl = `data:image/png;base64,${base64}`;
+      expect(extractBase64FromDataUrl(dataUrl)).toBe(base64);
+    });
+
+    it('should extract base64 from GIF data URL', () => {
+      const base64 = 'R0lGODlh';
+      const dataUrl = `data:image/gif;base64,${base64}`;
+      expect(extractBase64FromDataUrl(dataUrl)).toBe(base64);
+    });
+
+    it('should return unchanged raw base64 strings', () => {
+      const base64 = '/9j/4AAQSkZJRg';
+      expect(extractBase64FromDataUrl(base64)).toBe(base64);
+    });
+
+    it('should return unchanged HTTP URLs', () => {
+      const url = 'https://example.com/image.jpg';
+      expect(extractBase64FromDataUrl(url)).toBe(url);
+    });
+
+    it('should handle long base64 strings', () => {
+      const longBase64 =
+        '/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wgARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUAQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAGgP//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAQUCf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQMBAT8Bf//EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQIBAT8Bf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEABj8Cf//EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAT8hf//aAAwDAQACAAMAAAAQ/wD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/EH//xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/EH//xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/EH//2Q==';
+      const dataUrl = `data:image/jpeg;base64,${longBase64}`;
+      expect(extractBase64FromDataUrl(dataUrl)).toBe(longBase64);
+    });
+
+    it('should be idempotent - extracting from already extracted data returns same value', () => {
+      const base64 = '/9j/4AAQSkZJRg';
+      const extracted1 = extractBase64FromDataUrl(`data:image/jpeg;base64,${base64}`);
+      const extracted2 = extractBase64FromDataUrl(extracted1);
+      expect(extracted1).toBe(base64);
+      expect(extracted2).toBe(base64);
+    });
+
+    it('should handle malformed data URLs by returning them unchanged', () => {
+      const malformed = 'data:image/jpeg';
+      expect(extractBase64FromDataUrl(malformed)).toBe(malformed);
+    });
+
+    it('should handle empty strings', () => {
+      expect(extractBase64FromDataUrl('')).toBe('');
+    });
+  });
+
+  describe('integration scenarios', () => {
+    it('should handle Azure OpenAI use case (data URLs work natively)', () => {
+      const base64 = '/9j/4AAQSkZJRg';
+      const dataUrl = `data:image/jpeg;base64,${base64}`;
+
+      // Azure accepts data URLs directly
+      expect(isDataUrl(dataUrl)).toBe(true);
+      // No conversion needed
+    });
+
+    it('should handle Anthropic use case (needs base64 extraction)', () => {
+      const base64 = '/9j/4AAQSkZJRg';
+      const dataUrl = `data:image/jpeg;base64,${base64}`;
+
+      // Anthropic needs raw base64
+      const extracted = extractBase64FromDataUrl(dataUrl);
+      expect(extracted).toBe(base64);
+
+      // And MIME type
+      const parsed = parseDataUrl(dataUrl);
+      expect(parsed?.mimeType).toBe('image/jpeg');
+    });
+
+    it('should handle Google Gemini use case (needs base64 extraction)', () => {
+      const base64 = 'iVBORw0KGgo';
+      const dataUrl = `data:image/png;base64,${base64}`;
+
+      // Google needs raw base64 for inlineData.data
+      const extracted = extractBase64FromDataUrl(dataUrl);
+      expect(extracted).toBe(base64);
+
+      // And MIME type for inlineData.mimeType
+      const parsed = parseDataUrl(dataUrl);
+      expect(parsed?.mimeType).toBe('image/png');
+    });
+
+    it('should handle raw base64 passthrough (backwards compatibility)', () => {
+      const rawBase64 = '/9j/4AAQSkZJRg';
+
+      // Should not be detected as data URL
+      expect(isDataUrl(rawBase64)).toBe(false);
+
+      // Should pass through unchanged
+      expect(extractBase64FromDataUrl(rawBase64)).toBe(rawBase64);
+    });
+
+    it('should handle HTTP URL passthrough', () => {
+      const httpUrl = 'https://example.com/image.jpg';
+
+      // Should not be detected as data URL
+      expect(isDataUrl(httpUrl)).toBe(false);
+
+      // Should pass through unchanged
+      expect(extractBase64FromDataUrl(httpUrl)).toBe(httpUrl);
+    });
+  });
+
+  describe('toDataUri', () => {
+    it('should build a data URI from mime type and base64 data', () => {
+      expect(toDataUri('image/png', 'iVBORw0KGgo')).toBe('data:image/png;base64,iVBORw0KGgo');
+    });
+  });
+});

test/util/databaseDelete.test.ts

@@ -0,0 +1,135 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { updateSignalFileForDeletedEvals } from '../../src/database/signal';
+import { spansTable, tracesTable } from '../../src/database/tables';
+import { runDbMigrations } from '../../src/migrate';
+import Eval from '../../src/models/eval';
+import { TraceStore } from '../../src/tracing/store';
+import { deleteAllEvals, deleteEval, deleteEvals } from '../../src/util/database';
+import EvalFactory from '../factories/evalFactory';
+
+vi.mock('../../src/database/signal', async () => {
+  const actual = await vi.importActual('../../src/database/signal');
+  return {
+    ...actual,
+    updateSignalFile: vi.fn(),
+    updateSignalFileForDeletedEvals: vi.fn(),
+  };
+});
+
+describe('database eval deletion', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  beforeEach(async () => {
+    const db = await getDb();
+    await db.run('DELETE FROM spans');
+    await db.run('DELETE FROM traces');
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+  });
+
+  async function addTrace(evalId: string, traceId: string) {
+    const traceStore = new TraceStore();
+    await traceStore.createTrace({
+      traceId,
+      evaluationId: evalId,
+      testCaseId: 'test-case-id',
+    });
+    await traceStore.addSpans(traceId, [
+      {
+        spanId: `${traceId}-span`,
+        name: 'test-span',
+        startTime: 1,
+      },
+    ]);
+  }
+
+  it('deletes traces and spans for a single eval', async () => {
+    const eval_ = await EvalFactory.create();
+    await addTrace(eval_.id, 'trace-single');
+
+    await deleteEval(eval_.id);
+
+    const db = await getDb();
+    expect(await Eval.findById(eval_.id)).toBeUndefined();
+    expect(await db.select().from(tracesTable).all()).toHaveLength(0);
+    expect(await db.select().from(spansTable).all()).toHaveLength(0);
+    expect(updateSignalFileForDeletedEvals).toHaveBeenCalledWith([eval_.id]);
+  });
+
+  it('deletes only traces and spans for selected evals', async () => {
+    const eval1 = await EvalFactory.create();
+    const eval2 = await EvalFactory.create();
+    const eval3 = await EvalFactory.create();
+    await addTrace(eval1.id, 'trace-bulk-1');
+    await addTrace(eval2.id, 'trace-bulk-2');
+    await addTrace(eval3.id, 'trace-retained');
+
+    await deleteEvals([eval1.id, eval2.id]);
+
+    const db = await getDb();
+    expect(await Eval.findById(eval1.id)).toBeUndefined();
+    expect(await Eval.findById(eval2.id)).toBeUndefined();
+    expect(await Eval.findById(eval3.id)).toBeDefined();
+    expect(await db.select({ traceId: tracesTable.traceId }).from(tracesTable).all()).toEqual([
+      { traceId: 'trace-retained' },
+    ]);
+    expect(await db.select({ traceId: spansTable.traceId }).from(spansTable).all()).toEqual([
+      { traceId: 'trace-retained' },
+    ]);
+    expect(updateSignalFileForDeletedEvals).toHaveBeenCalledWith([eval1.id, eval2.id]);
+  });
+
+  it('does not emit a delete signal when called with an empty id list', async () => {
+    // An empty deletedEvalIds list is indistinguishable from "all evals deleted" on the
+    // client, so deleting zero evals must be a no-op rather than a spurious clear.
+    await deleteEvals([]);
+
+    expect(updateSignalFileForDeletedEvals).not.toHaveBeenCalled();
+  });
+
+  it('deletes traces and spans when deleting all evals', async () => {
+    const eval1 = await EvalFactory.create();
+    const eval2 = await EvalFactory.create();
+    await addTrace(eval1.id, 'trace-all-1');
+    await addTrace(eval2.id, 'trace-all-2');
+
+    await deleteAllEvals();
+
+    const db = await getDb();
+    expect(await Eval.getMany()).toHaveLength(0);
+    expect(await db.select().from(tracesTable).all()).toHaveLength(0);
+    expect(await db.select().from(spansTable).all()).toHaveLength(0);
+    expect(updateSignalFileForDeletedEvals).toHaveBeenCalledWith(undefined);
+  });
+
+  it('handles evals with no traces without error', async () => {
+    const eval_ = await EvalFactory.create();
+
+    await deleteEval(eval_.id);
+
+    expect(await Eval.findById(eval_.id)).toBeUndefined();
+  });
+
+  it('deletes every span when a trace has multiple spans', async () => {
+    const eval_ = await EvalFactory.create();
+    await addTrace(eval_.id, 'trace-multi-span');
+    await new TraceStore().addSpans('trace-multi-span', [
+      { spanId: 'extra-span', name: 'extra', startTime: 2 },
+    ]);
+
+    await deleteEval(eval_.id);
+
+    const db = await getDb();
+    expect(await db.select().from(spansTable).all()).toHaveLength(0);
+  });
+});

test/util/databaseStandalone.test.ts

@@ -0,0 +1,341 @@
+import { eq, sql } from 'drizzle-orm';
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { updateSignalFile } from '../../src/database/signal';
+import { evalsTable } from '../../src/database/tables';
+import { runDbMigrations } from '../../src/migrate';
+import Eval from '../../src/models/eval';
+import EvalResult from '../../src/models/evalResult';
+import { type CompletedPrompt, type Prompt, ResultFailureReason } from '../../src/types/index';
+import {
+  clearStandaloneEvalCache,
+  deleteEval,
+  getStandaloneEvals,
+  updateResult,
+} from '../../src/util/database';
+import {
+  getCachedStandaloneEvals,
+  getStandaloneEvalCacheKey,
+} from '../../src/util/standaloneEvalCache';
+
+vi.mock('../../src/database/signal', async () => {
+  const actual = await vi.importActual('../../src/database/signal');
+  return {
+    ...actual,
+    updateSignalFile: vi.fn(),
+  };
+});
+
+const completedPrompt: CompletedPrompt = {
+  raw: 'hello',
+  label: 'hello',
+  provider: 'test-provider',
+  metrics: {
+    score: 1,
+    testPassCount: 1,
+    testFailCount: 0,
+    testErrorCount: 0,
+    assertPassCount: 1,
+    assertFailCount: 0,
+    totalLatencyMs: 0,
+    tokenUsage: { total: 0, prompt: 0, completion: 0, cached: 0 },
+    namedScores: {},
+    namedScoresCount: {},
+    cost: 0,
+  },
+};
+
+const renderedPrompts: Prompt[] = [{ raw: 'hello', label: 'hello' }];
+
+// Mirrors the CASE expression in drizzle/0024_repair_eval_redteam_flags.sql.
+const isRedteamCase = sql`CASE
+  WHEN json_valid(${evalsTable.config}) AND json_type(${evalsTable.config}, '$.redteam') IS NOT NULL THEN 1
+  ELSE 0
+END`;
+
+async function createEvalWithPrompts(config: Partial<Parameters<typeof Eval.create>[0]>) {
+  return Eval.create(config as Parameters<typeof Eval.create>[0], renderedPrompts, {
+    completedPrompts: [completedPrompt],
+  });
+}
+
+async function resetEvalTables() {
+  const db = await getDb();
+  await db.run('DELETE FROM eval_results');
+  await db.run('DELETE FROM evals_to_datasets');
+  await db.run('DELETE FROM evals_to_prompts');
+  await db.run('DELETE FROM evals_to_tags');
+  await db.run('DELETE FROM evals');
+  clearStandaloneEvalCache();
+}
+
+async function setIsRedteam(evalId: string, value: boolean) {
+  const db = await getDb();
+  await db.update(evalsTable).set({ isRedteam: value }).where(eq(evalsTable.id, evalId)).run();
+}
+
+function expectStandaloneHistoryCached(options?: Parameters<typeof getStandaloneEvalCacheKey>[0]) {
+  expect(getCachedStandaloneEvals(getStandaloneEvalCacheKey(options))).toBeDefined();
+}
+
+describe('getStandaloneEvals', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(resetEvalTables);
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('returns isRedteam from the materialized column, not raw JSON inspection', async () => {
+    const redteamEval = await createEvalWithPrompts({ redteam: {} as any });
+    const regularEval = await createEvalWithPrompts({});
+
+    // Flip the column away from what the JSON config would imply; the query must trust the column.
+    await setIsRedteam(redteamEval.id, false);
+    await setIsRedteam(regularEval.id, true);
+
+    const rows = await getStandaloneEvals();
+    const byId = new Map(rows.map((row) => [row.evalId, row.isRedteam] as const));
+
+    expect(byId.get(redteamEval.id)).toBe(false);
+    expect(byId.get(regularEval.id)).toBe(true);
+  });
+
+  it('reflects flag transitions after updateResult rewrites config without stale cached history', async () => {
+    const eval_ = await createEvalWithPrompts({});
+    expect((await getStandaloneEvals()).find((row) => row.evalId === eval_.id)?.isRedteam).toBe(
+      false,
+    );
+
+    await updateResult(eval_.id, { redteam: {} as any });
+
+    expect((await getStandaloneEvals()).find((row) => row.evalId === eval_.id)?.isRedteam).toBe(
+      true,
+    );
+  });
+
+  it('drops deleted evals from cached history immediately', async () => {
+    const keep = await createEvalWithPrompts({});
+    const drop = await createEvalWithPrompts({});
+
+    const beforeIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(beforeIds).toEqual(expect.arrayContaining([keep.id, drop.id]));
+
+    await deleteEval(drop.id);
+
+    const afterIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(afterIds).toContain(keep.id);
+    expect(afterIds).not.toContain(drop.id);
+  });
+
+  it('includes newly created evals after history has been cached', async () => {
+    const first = await createEvalWithPrompts({});
+    const beforeIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    const limitedBeforeIds = (await getStandaloneEvals({ limit: 5 })).map((row) => row.evalId);
+    expect(beforeIds).toContain(first.id);
+    expect(limitedBeforeIds).toContain(first.id);
+    expectStandaloneHistoryCached();
+    expectStandaloneHistoryCached({ limit: 5 });
+
+    const second = await createEvalWithPrompts({});
+
+    const afterIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    const limitedAfterIds = (await getStandaloneEvals({ limit: 5 })).map((row) => row.evalId);
+    expect(afterIds).toContain(first.id);
+    expect(afterIds).toContain(second.id);
+    expect(limitedAfterIds).toContain(first.id);
+    expect(limitedAfterIds).toContain(second.id);
+  });
+
+  it('includes copied evals after history has been cached', async () => {
+    const source = await createEvalWithPrompts({});
+    const beforeIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(beforeIds).toContain(source.id);
+    expectStandaloneHistoryCached();
+
+    const persistedSource = await Eval.findById(source.id);
+    expect(persistedSource).toBeDefined();
+    const copy = await persistedSource!.copy('copy of source');
+
+    const afterIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(afterIds).toContain(source.id);
+    expect(afterIds).toContain(copy.id);
+    expect(updateSignalFile).toHaveBeenCalledWith(copy.id);
+  });
+
+  it('includes direct eval saves after history has been cached', async () => {
+    const eval_ = await createEvalWithPrompts({});
+    const beforeRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(beforeRow?.description).toBeNull();
+    expectStandaloneHistoryCached();
+
+    const persistedEval = await Eval.findById(eval_.id);
+    expect(persistedEval).toBeDefined();
+    persistedEval!.config.description = 'updated description';
+    await persistedEval!.save();
+
+    const afterRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(afterRow?.description).toBe('updated description');
+    expect(updateSignalFile).toHaveBeenCalledWith(eval_.id);
+  });
+
+  it('includes prompt changes after history has been cached', async () => {
+    const eval_ = await createEvalWithPrompts({});
+    const beforeRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(beforeRow?.raw).toBe('hello');
+    expectStandaloneHistoryCached();
+
+    await eval_.addPrompts([
+      {
+        ...completedPrompt,
+        raw: 'goodbye',
+        label: 'goodbye',
+      },
+    ]);
+
+    const afterRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(afterRow?.raw).toBe('goodbye');
+  });
+
+  it('includes result changes after history has been cached', async () => {
+    const eval_ = await createEvalWithPrompts({});
+    const beforeRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(beforeRow?.pluginFailCount['plugin-a']).toBeUndefined();
+    expectStandaloneHistoryCached();
+
+    await eval_.setResults([
+      new EvalResult({
+        id: 'set-results-history-row',
+        evalId: eval_.id,
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: {}, metadata: { pluginId: 'plugin-a' } },
+        prompt: renderedPrompts[0],
+        provider: { id: 'test-provider' },
+        response: { output: 'bad' },
+        gradingResult: null,
+        namedScores: {},
+        metadata: {},
+        success: false,
+        score: 0,
+        latencyMs: 1,
+        cost: 0,
+        failureReason: ResultFailureReason.ASSERT,
+      }),
+    ]);
+
+    const afterRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(afterRow?.pluginFailCount['plugin-a']).toBe(1);
+  });
+
+  it('includes incrementally added results after history has been cached', async () => {
+    const eval_ = await createEvalWithPrompts({});
+    const beforeRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(beforeRow?.pluginFailCount['plugin-add']).toBeUndefined();
+    expectStandaloneHistoryCached();
+
+    await eval_.addResult({
+      promptIdx: 0,
+      testIdx: 0,
+      testCase: { vars: {}, metadata: { pluginId: 'plugin-add' } },
+      promptId: 'prompt-add',
+      provider: { id: 'test-provider' },
+      prompt: renderedPrompts[0],
+      vars: {},
+      response: { output: 'bad' },
+      error: null,
+      failureReason: ResultFailureReason.ASSERT,
+      success: false,
+      score: 0,
+      latencyMs: 1,
+      gradingResult: null,
+      namedScores: {},
+      cost: 0,
+      metadata: {},
+    });
+
+    const afterRow = (await getStandaloneEvals()).find((row) => row.evalId === eval_.id);
+    expect(afterRow?.pluginFailCount['plugin-add']).toBe(1);
+  });
+
+  it('classifies redteam: null as redteam, matching the runtime predicate', async () => {
+    const eval_ = await createEvalWithPrompts({ redteam: null as any });
+
+    const db = await getDb();
+    const stored = await db
+      .select({ isRedteam: evalsTable.isRedteam })
+      .from(evalsTable)
+      .where(eq(evalsTable.id, eval_.id))
+      .get();
+    expect(stored?.isRedteam).toBe(true);
+
+    const row = (await getStandaloneEvals()).find((r) => r.evalId === eval_.id);
+    expect(row?.isRedteam).toBe(true);
+  });
+});
+
+describe('migration 0024 repair semantics', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(resetEvalTables);
+
+  it('classifies {redteam: null} as redteam via json_type (not json_extract)', async () => {
+    const db = await getDb();
+    const evalNullRedteam = await Eval.create({ redteam: null as any }, []);
+    const evalNoRedteam = await Eval.create({}, []);
+    const evalWithRedteam = await Eval.create({ redteam: {} as any }, []);
+
+    const classify = (id: string) =>
+      db
+        .select({
+          fromMigration: sql<number>`${isRedteamCase}`,
+          fromLegacyExtract: sql<number>`CASE
+            WHEN json_valid(${evalsTable.config}) AND json_extract(${evalsTable.config}, '$.redteam') IS NOT NULL THEN 1
+            ELSE 0
+          END`,
+        })
+        .from(evalsTable)
+        .where(eq(evalsTable.id, id))
+        .get();
+
+    const nullRow = await classify(evalNullRedteam.id);
+    const missingRow = await classify(evalNoRedteam.id);
+    const presentRow = await classify(evalWithRedteam.id);
+
+    expect(nullRow?.fromMigration).toBe(1);
+    expect(missingRow?.fromMigration).toBe(0);
+    expect(presentRow?.fromMigration).toBe(1);
+
+    // Legacy json_extract collapses {redteam: null} to non-redteam — the divergence this migration repairs.
+    expect(nullRow?.fromLegacyExtract).toBe(0);
+    expect(missingRow?.fromLegacyExtract).toBe(0);
+    expect(presentRow?.fromLegacyExtract).toBe(1);
+  });
+
+  it('idempotently repairs only stale rows', async () => {
+    const db = await getDb();
+    const evalWithRedteam = await Eval.create({ redteam: {} as any }, []);
+    const evalRegular = await Eval.create({}, []);
+
+    await setIsRedteam(evalWithRedteam.id, false);
+    await setIsRedteam(evalRegular.id, true);
+
+    const repairSql = sql`UPDATE evals SET is_redteam = ${isRedteamCase} WHERE is_redteam != ${isRedteamCase}`;
+
+    expect((await db.run(repairSql)).rowsAffected).toBe(2);
+    expect((await db.run(repairSql)).rowsAffected).toBe(0);
+
+    const rows = await db
+      .select({ id: evalsTable.id, isRedteam: evalsTable.isRedteam })
+      .from(evalsTable)
+      .all();
+    const byId = new Map(rows.map((row) => [row.id, row.isRedteam] as const));
+    expect(byId.get(evalWithRedteam.id)).toBe(true);
+    expect(byId.get(evalRegular.id)).toBe(false);
+  });
+});

test/util/databaseWrite.test.ts

@@ -0,0 +1,129 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { updateSignalFile } from '../../src/database/signal';
+import { datasetsTable, evalsTable, promptsTable, tagsTable } from '../../src/database/tables';
+import { runDbMigrations } from '../../src/migrate';
+import {
+  clearStandaloneEvalCache,
+  getStandaloneEvals,
+  writeResultsToDatabase,
+} from '../../src/util/database';
+import {
+  getCachedStandaloneEvals,
+  getStandaloneEvalCacheKey,
+} from '../../src/util/standaloneEvalCache';
+import { createEvaluateSummaryV2 } from '../factories/eval';
+
+vi.mock('../../src/database/signal', async () => {
+  const actual = await vi.importActual('../../src/database/signal');
+  return {
+    ...actual,
+    updateSignalFile: vi.fn(),
+  };
+});
+
+function expectWrittenHistoryCached() {
+  expect(getCachedStandaloneEvals(getStandaloneEvalCacheKey())).toBeDefined();
+}
+
+describe('writeResultsToDatabase', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  beforeEach(async () => {
+    const db = await getDb();
+    await db.run('DROP TRIGGER IF EXISTS fail_evals_to_tags_insert');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+    await db.run('DELETE FROM datasets');
+    await db.run('DELETE FROM prompts');
+    await db.run('DELETE FROM tags');
+    clearStandaloneEvalCache();
+  });
+
+  it('rolls back related rows when a dependent insert fails', async () => {
+    const db = await getDb();
+    await db.run(`
+      CREATE TRIGGER fail_evals_to_tags_insert
+      BEFORE INSERT ON evals_to_tags
+      BEGIN
+        SELECT RAISE(ABORT, 'forced tag failure');
+      END;
+    `);
+
+    await expect(
+      writeResultsToDatabase(
+        createEvaluateSummaryV2(),
+        {
+          tags: { suite: 'regression' },
+          tests: [],
+        },
+        new Date('2024-01-01T00:00:00.000Z'),
+      ),
+    ).rejects.toThrow();
+
+    await expect(db.select().from(evalsTable).all()).resolves.toHaveLength(0);
+    await expect(db.select().from(promptsTable).all()).resolves.toHaveLength(0);
+    await expect(db.select().from(datasetsTable).all()).resolves.toHaveLength(0);
+    await expect(db.select().from(tagsTable).all()).resolves.toHaveLength(0);
+  });
+
+  it('keeps cached history when a later write rolls back', async () => {
+    const firstEvalId = await writeResultsToDatabase(createEvaluateSummaryV2(), {}, new Date());
+    const beforeIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(beforeIds).toContain(firstEvalId);
+    expectWrittenHistoryCached();
+    const secondDate = new Date(Date.now() + 1000);
+
+    const db = await getDb();
+    await db.run(`
+      CREATE TRIGGER fail_evals_to_tags_insert
+      BEFORE INSERT ON evals_to_tags
+      BEGIN
+        SELECT RAISE(ABORT, 'forced tag failure');
+      END;
+    `);
+
+    await expect(
+      writeResultsToDatabase(
+        createEvaluateSummaryV2(),
+        {
+          tags: { suite: 'regression' },
+          tests: [],
+        },
+        secondDate,
+      ),
+    ).rejects.toThrow();
+
+    const cachedIds = getCachedStandaloneEvals(getStandaloneEvalCacheKey())?.map(
+      (row) => row.evalId,
+    );
+    expect(cachedIds).toContain(firstEvalId);
+  });
+
+  it('includes newly persisted evals after history has been cached', async () => {
+    const firstDate = new Date();
+    const firstEvalId = await writeResultsToDatabase(createEvaluateSummaryV2(), {}, firstDate);
+    const beforeIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(beforeIds).toContain(firstEvalId);
+    expectWrittenHistoryCached();
+
+    const secondEvalId = await writeResultsToDatabase(
+      createEvaluateSummaryV2(),
+      {},
+      new Date(firstDate.getTime() + 1000),
+    );
+
+    const afterIds = (await getStandaloneEvals()).map((row) => row.evalId);
+    expect(afterIds).toContain(firstEvalId);
+    expect(afterIds).toContain(secondEvalId);
+    expect(updateSignalFile).toHaveBeenCalledWith(secondEvalId);
+  });
+});

test/util/deleteAll.test.ts

@@ -0,0 +1,38 @@
+import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { updateSignalFileForDeletedEvals } from '../../src/database/signal';
+import { runDbMigrations } from '../../src/migrate';
+import Eval from '../../src/models/eval';
+import { deleteAllEvals } from '../../src/util/database';
+import EvalFactory from '../factories/evalFactory';
+
+vi.mock('../../src/database/signal', async () => {
+  const actual = await vi.importActual('../../src/database/signal');
+  return {
+    ...actual,
+    updateSignalFile: vi.fn(),
+    updateSignalFileForDeletedEvals: vi.fn(),
+  };
+});
+
+describe('delete all evals', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+  it('should delete all evals', async () => {
+    await EvalFactory.create();
+    await EvalFactory.create();
+    await EvalFactory.create();
+
+    const evals = await Eval.getMany();
+    expect(evals).toHaveLength(3);
+
+    await deleteAllEvals();
+
+    const evals2 = await Eval.getMany();
+    expect(evals2).toHaveLength(0);
+    expect(updateSignalFileForDeletedEvals).toHaveBeenCalledWith(undefined);
+  });
+});

test/util/env.test.ts

@@ -0,0 +1,211 @@
+import * as fs from 'fs';
+
+import dotenv from 'dotenv';
+import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from 'vitest';
+import logger from '../../src/logger';
+import * as configManage from '../../src/util/config/manage';
+import {
+  getConfigDirectoryPath,
+  refreshConfigDirectoryPathFromEnv,
+  setConfigDirectoryPath,
+} from '../../src/util/config/manage';
+import { setupEnv } from '../../src/util/env';
+import { mockProcessEnv } from './utils';
+
+vi.mock('fs', async () => {
+  const actual = await vi.importActual<typeof import('fs')>('fs');
+  return {
+    ...actual,
+    existsSync: vi.fn(),
+  };
+});
+
+describe('setupEnv', () => {
+  let originalEnv: typeof process.env;
+  let dotenvConfigSpy: MockInstance<
+    (options?: dotenv.DotenvConfigOptions) => dotenv.DotenvConfigOutput
+  >;
+  let loggerInfoSpy: MockInstance;
+
+  beforeEach(() => {
+    originalEnv = { ...process.env };
+    // Ensure NODE_ENV is not set at the start of each test
+    mockProcessEnv({ NODE_ENV: undefined });
+    mockProcessEnv({ PROMPTFOO_CONFIG_DIR: undefined });
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+    // Spy on dotenv.config to verify it's called with the right parameters
+    dotenvConfigSpy = vi.spyOn(dotenv, 'config').mockImplementation(() => ({ parsed: {} }));
+    loggerInfoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);
+    // Mock file existence check - default to true for backward compat with existing tests
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    mockProcessEnv(originalEnv, { clear: true });
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+    vi.resetAllMocks();
+  });
+
+  it('should call dotenv.config with quiet=true when envPath is undefined', () => {
+    setupEnv(undefined);
+
+    expect(dotenvConfigSpy).toHaveBeenCalledTimes(1);
+    expect(dotenvConfigSpy).toHaveBeenCalledWith({ quiet: true });
+  });
+
+  it('should call dotenv.config with path, override=true, and quiet=true when envPath is specified', () => {
+    const testEnvPath = '.env.test';
+
+    setupEnv(testEnvPath);
+
+    expect(dotenvConfigSpy).toHaveBeenCalledTimes(1);
+    expect(dotenvConfigSpy).toHaveBeenCalledWith({
+      path: testEnvPath,
+      override: true,
+      quiet: true,
+    });
+    expect(loggerInfoSpy).toHaveBeenCalledWith('Loading environment variables from .env.test');
+  });
+
+  it('should load environment variables with override when specified env file has conflicting values', () => {
+    // Mock dotenv.config to simulate loading variables
+    dotenvConfigSpy.mockImplementation((options?: dotenv.DotenvConfigOptions) => {
+      if (options?.path === '.env.production') {
+        if (options.override) {
+          mockProcessEnv({ NODE_ENV: 'production' });
+        } else if (!process.env.NODE_ENV) {
+          mockProcessEnv({ NODE_ENV: 'production' });
+        }
+      } else {
+        // Default .env file
+        if (!process.env.NODE_ENV) {
+          mockProcessEnv({ NODE_ENV: 'development' });
+        }
+      }
+      return { parsed: {} };
+    });
+
+    // First load the default .env (setting NODE_ENV to 'development')
+    setupEnv(undefined);
+    expect(process.env.NODE_ENV).toBe('development');
+
+    // Then load .env.production with override (should change NODE_ENV to 'production')
+    setupEnv('.env.production');
+    expect(process.env.NODE_ENV).toBe('production');
+  });
+
+  it('should refresh the config directory after early env loading and freeze later changes', () => {
+    const refreshConfigDirectorySpy = vi.spyOn(configManage, 'refreshConfigDirectoryPathFromEnv');
+    dotenvConfigSpy.mockImplementation(() => {
+      mockProcessEnv({ PROMPTFOO_CONFIG_DIR: '/early/config' });
+      return { parsed: {} };
+    });
+
+    setupEnv('.env.early', { refreshConfigDirectory: true });
+    expect(refreshConfigDirectorySpy).toHaveBeenCalledTimes(1);
+    expect(getConfigDirectoryPath()).toBe('/early/config');
+
+    mockProcessEnv({ PROMPTFOO_CONFIG_DIR: '/late/config' });
+    expect(getConfigDirectoryPath()).toBe('/early/config');
+  });
+
+  describe('multi-file support', () => {
+    it('should call dotenv.config with array of paths when multiple files specified', () => {
+      const paths = ['.env', '.env.local'];
+
+      setupEnv(paths);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledTimes(1);
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({
+        path: paths,
+        override: true,
+        quiet: true,
+      });
+      expect(loggerInfoSpy).toHaveBeenCalledWith(
+        'Loading environment variables from: .env, .env.local',
+      );
+    });
+
+    it('should call dotenv.config with single path (not array) when one file specified as array', () => {
+      const paths = ['.env'];
+
+      setupEnv(paths);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledTimes(1);
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({
+        path: '.env',
+        override: true,
+        quiet: true,
+      });
+      expect(loggerInfoSpy).toHaveBeenCalledWith('Loading environment variables from .env');
+    });
+
+    it('should throw error when specified file does not exist', () => {
+      vi.mocked(fs.existsSync).mockReturnValue(false);
+
+      expect(() => setupEnv('.env.missing')).toThrow('Environment file not found: .env.missing');
+    });
+
+    it('should throw error when any file in array does not exist', () => {
+      vi.mocked(fs.existsSync).mockImplementation((p) => p === '.env');
+
+      expect(() => setupEnv(['.env', '.env.missing'])).toThrow(
+        'Environment file not found: .env.missing',
+      );
+    });
+
+    it('should validate all files exist before calling dotenv.config', () => {
+      vi.mocked(fs.existsSync).mockImplementation((p) => p === '.env');
+
+      expect(() => setupEnv(['.env', '.env.missing'])).toThrow();
+      expect(dotenvConfigSpy).not.toHaveBeenCalled();
+    });
+
+    it('should filter out empty strings from array', () => {
+      setupEnv(['', '.env', '  ']);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({
+        path: '.env',
+        override: true,
+        quiet: true,
+      });
+      expect(loggerInfoSpy).toHaveBeenCalledWith('Loading environment variables from .env');
+    });
+
+    it('should call default dotenv.config when array contains only empty strings', () => {
+      setupEnv(['', '  ', '']);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({ quiet: true });
+    });
+
+    it('should call default dotenv.config when given empty array', () => {
+      setupEnv([]);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({ quiet: true });
+    });
+
+    it('should expand comma-separated values within array elements', () => {
+      // This simulates what Commander passes when using --env-file .env,.env.local
+      setupEnv(['.env,.env.local']);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledTimes(1);
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({
+        path: ['.env', '.env.local'],
+        override: true,
+        quiet: true,
+      });
+    });
+
+    it('should handle mixed array with some comma-separated and some individual paths', () => {
+      setupEnv(['.env,.env.local', '.env.production']);
+
+      expect(dotenvConfigSpy).toHaveBeenCalledWith({
+        path: ['.env', '.env.local', '.env.production'],
+        override: true,
+        quiet: true,
+      });
+    });
+  });
+});

test/util/errors/index.test.ts

@@ -0,0 +1,129 @@
+import fs from 'fs';
+
+import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from 'vitest';
+import logger from '../../../src/logger';
+import { printErrorInformation } from '../../../src/util/errors';
+
+vi.mock('fs');
+
+describe('printErrorInformation', () => {
+  let infoSpy: MockInstance<typeof logger.info>;
+  let debugSpy: MockInstance<typeof logger.debug>;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);
+    debugSpy = vi.spyOn(logger, 'debug').mockImplementation(() => logger);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('does nothing when no errorLogFile is provided', () => {
+    printErrorInformation();
+
+    expect(infoSpy).not.toHaveBeenCalled();
+    expect(fs.statSync).not.toHaveBeenCalled();
+  });
+
+  it('does nothing when errorLogFile is an empty string', () => {
+    printErrorInformation('');
+
+    expect(infoSpy).not.toHaveBeenCalled();
+    expect(fs.statSync).not.toHaveBeenCalled();
+  });
+
+  it('does nothing when the error log file does not exist (ENOENT)', () => {
+    const enoent = new Error('ENOENT') as NodeJS.ErrnoException;
+    enoent.code = 'ENOENT';
+    vi.mocked(fs.statSync).mockImplementation(() => {
+      throw enoent;
+    });
+
+    printErrorInformation('/missing/error.log');
+
+    expect(infoSpy).not.toHaveBeenCalled();
+    // ENOENT should not be logged as debug noise
+    expect(debugSpy).not.toHaveBeenCalled();
+  });
+
+  it('does nothing when the error log file exists but is empty', () => {
+    vi.mocked(fs.statSync).mockReturnValue({
+      isFile: () => true,
+      size: 0,
+    } as fs.Stats);
+
+    printErrorInformation('/logs/error.log');
+
+    expect(infoSpy).not.toHaveBeenCalled();
+  });
+
+  it('does nothing when the path exists but is not a file', () => {
+    vi.mocked(fs.statSync).mockReturnValue({
+      isFile: () => false,
+      size: 1024,
+    } as fs.Stats);
+
+    printErrorInformation('/logs');
+
+    expect(infoSpy).not.toHaveBeenCalled();
+  });
+
+  it('logs a debug message for unexpected fs errors (e.g. EACCES)', () => {
+    const eacces = new Error('EACCES') as NodeJS.ErrnoException;
+    eacces.code = 'EACCES';
+    vi.mocked(fs.statSync).mockImplementation(() => {
+      throw eacces;
+    });
+
+    printErrorInformation('/logs/error.log');
+
+    expect(infoSpy).not.toHaveBeenCalled();
+    expect(debugSpy).toHaveBeenCalledTimes(1);
+    expect(debugSpy).toHaveBeenCalledWith(
+      expect.stringContaining(
+        '[errorFileHasContents] Error checking if file has contents: /logs/error.log',
+      ),
+      { error: eacces },
+    );
+  });
+
+  it('prints only the error log path when no debug log is provided', () => {
+    vi.mocked(fs.statSync).mockReturnValue({
+      isFile: () => true,
+      size: 42,
+    } as fs.Stats);
+
+    printErrorInformation('/logs/error.log');
+
+    expect(infoSpy).toHaveBeenCalledTimes(1);
+    const message = infoSpy.mock.calls[0][0] as string;
+    expect(message).toContain('There were some errors during the operation');
+    expect(message).toContain('/logs/error.log');
+    expect(message).not.toContain('Debug log:');
+  });
+
+  it('prints both error and debug log paths when both are provided', () => {
+    vi.mocked(fs.statSync).mockReturnValue({
+      isFile: () => true,
+      size: 42,
+    } as fs.Stats);
+
+    printErrorInformation('/logs/error.log', '/logs/debug.log');
+
+    expect(infoSpy).toHaveBeenCalledTimes(1);
+    const message = infoSpy.mock.calls[0][0] as string;
+    expect(message).toContain('Error log:');
+    expect(message).toContain('/logs/error.log');
+    expect(message).toContain('Debug log:');
+    expect(message).toContain('/logs/debug.log');
+  });
+
+  it('ignores debugLogFile when errorLogFile is missing/empty', () => {
+    printErrorInformation(undefined, '/logs/debug.log');
+
+    expect(infoSpy).not.toHaveBeenCalled();
+    expect(fs.statSync).not.toHaveBeenCalled();
+  });
+});

test/util/eval/filterFailingBug.test.ts

@@ -0,0 +1,734 @@
+/**
+ * Test for the --filter-failing bug where test.vars gets mutated
+ * during evaluation, causing mismatches when trying to filter failing tests.
+ *
+ * Bug 1: When evaluation adds runtime vars like _conversation to test.vars,
+ * it mutates the original test object. This mutated test is stored in results.
+ * On subsequent runs with --filter-failing, freshly loaded tests don't have
+ * these runtime vars, so deepEqual comparison fails.
+ *
+ * Bug 2: Multi-turn strategy providers (GOAT, Crescendo, SIMBA) add sessionId
+ * to vars during execution, causing the same mismatch issue.
+ *
+ * Fix: We filter out known runtime vars (_conversation, sessionId) during test
+ * matching in resultIsForTestCase, and create a shallow copy of test.vars in
+ * runEval to prevent mutation by reference.
+ */
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import Eval from '../../../src/models/eval';
+import { ResultFailureReason } from '../../../src/types/index';
+import { filterTests } from '../../../src/util/eval/filterTests';
+
+import type { TestSuite } from '../../../src/types/index';
+
+vi.mock('../../../src/models/eval', () => ({
+  default: {
+    findById: vi.fn(),
+  },
+}));
+
+describe('filterTests - vars mutation bug', () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('should match tests even when stored results have additional runtime vars', async () => {
+    // Simulate a test suite as it would be loaded from config
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'test1',
+          vars: { input: 'hello', language: 'en' },
+          assert: [],
+        },
+        {
+          description: 'test2',
+          vars: { input: 'goodbye', language: 'en' },
+          assert: [],
+        },
+        {
+          description: 'test3',
+          vars: { input: 'thanks', language: 'en' },
+          assert: [],
+        },
+      ],
+    };
+
+    // Simulate stored results where test.vars was mutated with _conversation
+    // This is what happens in the bug - the runtime adds _conversation to vars
+    const mockEval = {
+      id: 'eval-123',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // This test failed and has the original vars
+            vars: { input: 'hello', language: 'en' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'test1',
+              // Bug: stored testCase.vars has _conversation added
+              vars: { input: 'hello', language: 'en', _conversation: [] },
+              assert: [],
+            },
+          },
+          {
+            // This test also failed (assertion failure, not error)
+            vars: { input: 'thanks', language: 'en' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'test3',
+              // Bug: stored testCase.vars has _conversation added
+              vars: { input: 'thanks', language: 'en', _conversation: [] },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // When filtering failing tests, it should match based on the original vars
+    // not fail because of the extra _conversation property
+    const result = await filterTests(mockTestSuite, { failing: 'eval-123' });
+
+    // Should find both failing tests (both have ASSERT failure reason)
+    expect(result).toHaveLength(2);
+    expect(result.map((t) => t.description)).toEqual(['test1', 'test3']);
+  });
+
+  it('should include both failures and errors when using --filter-failing', async () => {
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        { description: 'test1', vars: { input: 'a' }, assert: [] },
+        { description: 'test2', vars: { input: 'b' }, assert: [] },
+        { description: 'test3', vars: { input: 'c' }, assert: [] },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-errors',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { input: 'a' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT, // Actual failure
+            testCase: { description: 'test1', vars: { input: 'a' }, assert: [] },
+          },
+          {
+            vars: { input: 'b' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR, // Error
+            testCase: { description: 'test2', vars: { input: 'b' }, assert: [] },
+          },
+          {
+            vars: { input: 'c' },
+            success: true, // Success
+            testCase: { description: 'test3', vars: { input: 'c' }, assert: [] },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // --filter-failing should return both failures AND errors (all non-successful results)
+    const result = await filterTests(mockTestSuite, { failing: 'eval-errors' });
+
+    // Should find test1 (ASSERT failure) and test2 (ERROR), but not test3 (success)
+    expect(result).toHaveLength(2);
+    expect(result.map((t) => t.description).sort()).toEqual(['test1', 'test2']);
+  });
+
+  it('should exclude error results when using --filter-failing-only', async () => {
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        { description: 'test1', vars: { input: 'a' }, assert: [] },
+        { description: 'test2', vars: { input: 'b' }, assert: [] },
+        { description: 'test3', vars: { input: 'c' }, assert: [] },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-errors',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { input: 'a' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT, // Actual failure
+            testCase: { description: 'test1', vars: { input: 'a' }, assert: [] },
+          },
+          {
+            vars: { input: 'b' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR, // Error, not failure
+            testCase: { description: 'test2', vars: { input: 'b' }, assert: [] },
+          },
+          {
+            vars: { input: 'c' },
+            success: true, // Success
+            testCase: { description: 'test3', vars: { input: 'c' }, assert: [] },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // --filter-failing-only should only return actual failures, not errors
+    const result = await filterTests(mockTestSuite, { failingOnly: 'eval-errors' });
+
+    // Should only find test1 (the one with ASSERT failure), not test2 (error)
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('test1');
+  });
+
+  it('should match tests even when stored results have sessionId added by GOAT/Crescendo providers', async () => {
+    // Simulate a redteam test suite with GOAT strategy
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'goat-test-1',
+          vars: { prompt: 'test attack prompt', goal: 'test goal' },
+          assert: [],
+          metadata: { pluginId: 'harmful', strategyId: 'goat' },
+        },
+        {
+          description: 'goat-test-2',
+          vars: { prompt: 'another attack prompt', goal: 'another goal' },
+          assert: [],
+          metadata: { pluginId: 'harmful', strategyId: 'goat' },
+        },
+      ],
+    };
+
+    // Simulate stored results where vars was mutated with sessionId by GOAT provider
+    const mockEval = {
+      id: 'eval-456',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // GOAT provider added sessionId during multi-turn attack
+            vars: {
+              prompt: 'test attack prompt',
+              goal: 'test goal',
+              sessionId: 'goat-session-abc',
+            },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: {
+              description: 'goat-test-1',
+              // Stored testCase.vars has sessionId added by GOAT provider
+              vars: {
+                prompt: 'test attack prompt',
+                goal: 'test goal',
+                sessionId: 'goat-session-abc',
+              },
+              assert: [],
+              metadata: { pluginId: 'harmful', strategyId: 'goat' },
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // When filtering error tests, it should match based on the original vars
+    // not fail because of the extra sessionId property added by GOAT
+    const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-456' });
+
+    // Should find the error test even though sessionId was added
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('goat-test-1');
+  });
+
+  it('should match tests with both _conversation and sessionId runtime vars', async () => {
+    // Combined scenario with both runtime vars
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'combined-test',
+          vars: { input: 'hello' },
+          assert: [],
+        },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-789',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // Both runtime vars added during evaluation
+            vars: {
+              input: 'hello',
+              _conversation: [{ role: 'user', content: 'hi' }],
+              sessionId: 'session-xyz',
+            },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'combined-test',
+              vars: {
+                input: 'hello',
+                _conversation: [{ role: 'user', content: 'hi' }],
+                sessionId: 'session-xyz',
+              },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 1,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    const result = await filterTests(mockTestSuite, { failing: 'eval-789' });
+
+    // Should match even with both runtime vars present
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('combined-test');
+  });
+
+  it('should match tests when stored results have defaultTest.vars merged', async () => {
+    // Simulate a test suite with defaultTest.vars (as it would be loaded from config)
+    // Note: Fresh tests don't have defaultTest.vars merged yet - that happens in prepareTests
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      defaultTest: {
+        vars: {
+          systemPrompt: 'You are a helpful assistant',
+        },
+      },
+      tests: [
+        {
+          description: 'test-with-defaults',
+          vars: { prompt: 'first test' },
+          assert: [],
+        },
+        {
+          description: 'test-that-passed',
+          vars: { prompt: 'second test' },
+          assert: [],
+        },
+      ],
+    };
+
+    // Stored results have defaultTest.vars merged (as done by prepareTests in evaluator)
+    const mockEval = {
+      id: 'eval-default-vars',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // Stored vars have defaultTest.vars merged
+            vars: { systemPrompt: 'You are a helpful assistant', prompt: 'first test' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'test-with-defaults',
+              vars: { systemPrompt: 'You are a helpful assistant', prompt: 'first test' },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // filterTests should merge defaultTest.vars before comparison
+    const result = await filterTests(mockTestSuite, { failing: 'eval-default-vars' });
+
+    // Should find the failing test even though fresh test.vars doesn't have systemPrompt
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('test-with-defaults');
+  });
+
+  it('should match old results that lack defaultTest.vars via fallback', async () => {
+    // This test verifies the fallback matching for old results:
+    // - Config has defaultTest.vars defined
+    // - But stored results were created without defaultTest.vars merged
+    // - The fallback tries matching without defaults, which succeeds
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      defaultTest: {
+        vars: {
+          systemPrompt: 'You are a helpful assistant',
+          context: 'General knowledge',
+        },
+      },
+      tests: [
+        {
+          description: 'emoji-test',
+          vars: { prompt: '😊' + String.fromCodePoint(0xfe00) }, // Emoji encoded
+          assert: [],
+        },
+      ],
+    };
+
+    // Stored results DON'T have defaultTest.vars (old version or bug)
+    const mockEval = {
+      id: 'eval-missing-defaults',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // Stored vars only have the test-specific vars, NOT the defaults
+            vars: { prompt: '😊' + String.fromCodePoint(0xfe00) },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: {
+              description: 'emoji-test',
+              vars: { prompt: '😊' + String.fromCodePoint(0xfe00) },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // With the fallback, this SHOULD match because:
+    // 1. First try: testCase.vars after merge: { systemPrompt, context, prompt } vs result.vars: { prompt } - NO MATCH
+    // 2. Fallback: testCase.vars without merge: { prompt } vs result.vars: { prompt } - MATCH!
+    const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-missing-defaults' });
+
+    // Should find the test via fallback matching
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('emoji-test');
+  });
+
+  it('should match when defaultTest.vars is empty or undefined', async () => {
+    // When defaultTest has no vars, the merge should not affect matching
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      defaultTest: {
+        // No vars defined, or vars is empty
+      },
+      tests: [
+        {
+          description: 'basic-test',
+          vars: { prompt: 'test prompt' },
+          assert: [],
+        },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-no-defaults',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { prompt: 'test prompt', sessionId: 'runtime-session' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: {
+              description: 'basic-test',
+              vars: { prompt: 'test prompt' },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // Should match because:
+    // - testCase.vars after merge (no defaults): { prompt }
+    // - result.vars after filtering sessionId: { prompt }
+    const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-no-defaults' });
+
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('basic-test');
+  });
+
+  it('should combine failingOnly and errorsOnly filters (union) when both are provided', async () => {
+    // When both --filter-failing-only and --filter-errors-only are used, combine results
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'error-test',
+          vars: { prompt: 'error prompt' },
+          assert: [],
+        },
+        {
+          description: 'failure-test',
+          vars: { prompt: 'failure prompt' },
+          assert: [],
+        },
+        {
+          description: 'passing-test',
+          vars: { prompt: 'passing prompt' },
+          assert: [],
+        },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-combined',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { prompt: 'error prompt' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: { description: 'error-test', vars: { prompt: 'error prompt' }, assert: [] },
+          },
+          {
+            vars: { prompt: 'failure prompt' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'failure-test',
+              vars: { prompt: 'failure prompt' },
+              assert: [],
+            },
+          },
+          {
+            vars: { prompt: 'passing prompt' },
+            success: true,
+            failureReason: ResultFailureReason.NONE,
+            testCase: {
+              description: 'passing-test',
+              vars: { prompt: 'passing prompt' },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // Using both filters should return union of failingOnly + errors (but not passing)
+    const result = await filterTests(mockTestSuite, {
+      failingOnly: 'eval-combined',
+      errorsOnly: 'eval-combined',
+    });
+
+    // Should find both error-test and failure-test (union), but not passing-test
+    expect(result).toHaveLength(2);
+    const descriptions = result.map((t) => t.description).sort();
+    expect(descriptions).toEqual(['error-test', 'failure-test']);
+  });
+});

test/util/eval/filterPrompts.test.ts

@@ -0,0 +1,162 @@
+import { describe, expect, it } from 'vitest';
+import { filterPrompts } from '../../../src/util/eval/filterPrompts';
+
+import type { Prompt } from '../../../src/types/index';
+
+describe('filterPrompts', () => {
+  const mockPrompts: Prompt[] = [
+    {
+      id: 'prompt-1',
+      raw: 'Hello {{name}}',
+      label: 'Greeting Prompt',
+    },
+    {
+      id: 'prompt-2',
+      raw: 'Goodbye {{name}}',
+      label: 'Farewell Prompt',
+    },
+    {
+      id: 'prompt-3',
+      raw: 'How are you?',
+      label: 'Question Prompt',
+    },
+    {
+      id: 'prompt-4',
+      raw: 'System message',
+      label: 'System:Admin',
+    },
+    {
+      id: 'prompt-5',
+      raw: 'No label here',
+      label: '', // Empty label
+    },
+  ];
+
+  it('should return all prompts if no filter is provided', () => {
+    const result = filterPrompts(mockPrompts);
+    expect(result).toEqual(mockPrompts);
+  });
+
+  it('should filter prompts by exact label match', () => {
+    const result = filterPrompts(mockPrompts, 'Greeting Prompt');
+    expect(result).toHaveLength(1);
+    expect(result[0].label).toBe('Greeting Prompt');
+  });
+
+  it('should filter prompts by partial label match', () => {
+    const result = filterPrompts(mockPrompts, 'Prompt');
+    expect(result).toHaveLength(3);
+    expect(result.map((p) => p.label)).toEqual([
+      'Greeting Prompt',
+      'Farewell Prompt',
+      'Question Prompt',
+    ]);
+  });
+
+  it('should filter prompts by id', () => {
+    const result = filterPrompts(mockPrompts, 'prompt-1');
+    expect(result).toHaveLength(1);
+    expect(result[0].id).toBe('prompt-1');
+  });
+
+  it('should filter prompts by id pattern', () => {
+    const result = filterPrompts(mockPrompts, 'prompt-[12]');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.id)).toEqual(['prompt-1', 'prompt-2']);
+  });
+
+  it('should match on either id or label', () => {
+    const mixedPrompts: Prompt[] = [
+      { id: 'greeting-v1', raw: 'Hello', label: 'Greeting' },
+      { id: 'farewell-v1', raw: 'Goodbye', label: 'Farewell' },
+    ];
+    const result1 = filterPrompts(mixedPrompts, 'v1');
+    expect(result1).toHaveLength(2);
+    const result2 = filterPrompts(mixedPrompts, 'Greeting');
+    expect(result2).toHaveLength(1);
+    expect(result2[0].label).toBe('Greeting');
+  });
+
+  it('should handle regex patterns', () => {
+    const result = filterPrompts(mockPrompts, '(Greeting|Farewell)');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.label)).toEqual(['Greeting Prompt', 'Farewell Prompt']);
+  });
+
+  it('should handle case-sensitive regex', () => {
+    const result = filterPrompts(mockPrompts, 'greeting');
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle case-sensitive matching (default behavior)', () => {
+    // JavaScript RegExp is case-sensitive by default
+    // Users can use [Gg] or similar patterns for case-insensitive matching
+    const result = filterPrompts(mockPrompts, '[Gg]reeting');
+    expect(result).toHaveLength(1);
+    expect(result[0].label).toBe('Greeting Prompt');
+  });
+
+  it('should return empty array if no prompts match filter', () => {
+    const result = filterPrompts(mockPrompts, 'nonexistent');
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle prompts with empty labels', () => {
+    const result = filterPrompts(mockPrompts, 'prompt-');
+    expect(result).toHaveLength(5);
+  });
+
+  it('should handle prompts without label property', () => {
+    const promptsWithoutLabel: Prompt[] = [
+      {
+        id: 'test-1',
+        raw: 'Test',
+        label: 'Valid Label',
+      },
+      {
+        id: 'test-2',
+        raw: 'Test 2',
+        label: undefined as any, // Simulate missing label
+      },
+    ];
+    const result1 = filterPrompts(promptsWithoutLabel, 'Valid');
+    expect(result1).toHaveLength(1);
+    expect(result1[0].label).toBe('Valid Label');
+    const result2 = filterPrompts(promptsWithoutLabel, 'test-');
+    expect(result2).toHaveLength(2);
+  });
+
+  it('should handle special regex characters in labels', () => {
+    const result = filterPrompts(mockPrompts, 'System:Admin');
+    expect(result).toHaveLength(1);
+    expect(result[0].label).toBe('System:Admin');
+  });
+
+  it('should handle complex regex patterns', () => {
+    const result = filterPrompts(mockPrompts, '^(Greeting|Question).*');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.label)).toEqual(['Greeting Prompt', 'Question Prompt']);
+  });
+
+  it('should handle empty prompts array', () => {
+    const result = filterPrompts([], 'anything');
+    expect(result).toEqual([]);
+  });
+
+  it('should throw on invalid regex with helpful error message', () => {
+    expect(() => filterPrompts(mockPrompts, '[invalid')).toThrow(
+      'Invalid regex pattern for --filter-prompts: "[invalid"',
+    );
+  });
+
+  it('should throw on invalid regex with multiple patterns', () => {
+    expect(() => filterPrompts(mockPrompts, '((')).toThrow(
+      'Invalid regex pattern for --filter-prompts',
+    );
+  });
+
+  it('should filter with word boundaries', () => {
+    const result = filterPrompts(mockPrompts, '\\bPrompt\\b');
+    expect(result).toHaveLength(3);
+  });
+});

test/util/eval/filterProviders.test.ts

@@ -0,0 +1,316 @@
+import { describe, expect, it } from 'vitest';
+import {
+  filterProviderConfigs,
+  filterProviders,
+  getPersistedProviderFilterOptions,
+  getProviderFilterRegexError,
+} from '../../../src/util/eval/filterProviders';
+
+import type { ApiProvider, TestSuiteConfig } from '../../../src/types/index';
+import type { ProviderOptions, ProviderOptionsMap } from '../../../src/types/providers';
+
+describe('getPersistedProviderFilterOptions', () => {
+  it('converts a persisted filter into config resolution options', () => {
+    expect(getPersistedProviderFilterOptions('selected-target')).toEqual({
+      filterProviders: 'selected-target',
+    });
+  });
+
+  it.each([undefined, null])('treats a missing persisted filter as no filter: %j', (value) => {
+    expect(getPersistedProviderFilterOptions(value)).toEqual({});
+  });
+
+  it.each(['', 42, {}, ['echo']])('fails closed on a tampered persisted filter: %j', (value) => {
+    expect(() => getPersistedProviderFilterOptions(value)).toThrow(
+      'Stored provider filter is invalid',
+    );
+  });
+});
+
+describe('getProviderFilterRegexError', () => {
+  it('returns undefined for a valid pattern', () => {
+    expect(getProviderFilterRegexError('selected-.*')).toBeUndefined();
+  });
+
+  it('returns the reason for an invalid pattern', () => {
+    expect(getProviderFilterRegexError('[')).toContain('Invalid regular expression');
+  });
+});
+
+describe('filterProviders', () => {
+  const mockProviders: ApiProvider[] = [
+    {
+      id: () => 'openai:gpt-4',
+      label: 'GPT-4',
+      callApi: async () => ({ output: '' }),
+    },
+    {
+      id: () => 'openai:gpt-3.5-turbo',
+      label: 'GPT-3.5',
+      callApi: async () => ({ output: '' }),
+    },
+    {
+      id: () => 'anthropic:claude-2',
+      label: 'Claude',
+      callApi: async () => ({ output: '' }),
+    },
+    {
+      id: () => 'custom:provider',
+      callApi: async () => ({ output: '' }),
+      // No label
+    },
+  ];
+
+  it('should return all providers if no filter is provided', () => {
+    const result = filterProviders(mockProviders);
+    expect(result).toEqual(mockProviders);
+  });
+
+  it('should filter providers by ID', () => {
+    const result = filterProviders(mockProviders, 'openai');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.id())).toEqual(['openai:gpt-4', 'openai:gpt-3.5-turbo']);
+  });
+
+  it('should filter providers by label', () => {
+    const result = filterProviders(mockProviders, 'GPT');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.label)).toEqual(['GPT-4', 'GPT-3.5']);
+  });
+
+  it('should handle providers without labels', () => {
+    const result = filterProviders(mockProviders, 'custom');
+    expect(result).toHaveLength(1);
+    expect(result[0].id()).toBe('custom:provider');
+  });
+
+  it('should handle regex patterns', () => {
+    const result = filterProviders(mockProviders, '(gpt|claude)');
+    expect(result).toHaveLength(3);
+    expect(result.map((p) => p.id())).toEqual([
+      'openai:gpt-4',
+      'openai:gpt-3.5-turbo',
+      'anthropic:claude-2',
+    ]);
+  });
+
+  it('should return empty array if no providers match filter', () => {
+    const result = filterProviders(mockProviders, 'nonexistent');
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle case sensitivity', () => {
+    const result = filterProviders(mockProviders, 'GPT');
+    expect(result).toHaveLength(2);
+  });
+});
+
+describe('filterProviderConfigs', () => {
+  describe('string providers', () => {
+    it('should return the string if it matches the filter', () => {
+      const result = filterProviderConfigs('openai:gpt-4', 'openai');
+      expect(result).toBe('openai:gpt-4');
+    });
+
+    it('should return empty array if string does not match', () => {
+      const result = filterProviderConfigs('openai:gpt-4', 'anthropic');
+      expect(result).toEqual([]);
+    });
+
+    it('should handle regex patterns for strings', () => {
+      const result = filterProviderConfigs('openai:gpt-4', 'gpt-[0-9]');
+      expect(result).toBe('openai:gpt-4');
+    });
+  });
+
+  describe('function providers', () => {
+    it('should filter function provider by label', () => {
+      const fn = Object.assign(async () => ({ output: '' }), { label: 'MyFunction' });
+      const result = filterProviderConfigs(fn, 'MyFunction');
+      expect(result).toBe(fn);
+    });
+
+    it('should filter function provider without label by default id', () => {
+      const fn = async () => ({ output: '' });
+      const result = filterProviderConfigs(fn, 'custom-function');
+      expect(result).toBe(fn);
+    });
+
+    it('should return empty array if function label does not match', () => {
+      const fn = Object.assign(async () => ({ output: '' }), { label: 'MyFunction' });
+      const result = filterProviderConfigs(fn, 'OtherFunction');
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe('array of providers', () => {
+    it('should return all providers if no filter is provided', () => {
+      const providers = ['openai:gpt-4', 'anthropic:claude-2'];
+      const result = filterProviderConfigs(providers);
+      expect(result).toEqual(providers);
+    });
+
+    it('should filter string providers in array', () => {
+      const providers = ['openai:gpt-4', 'openai:gpt-3.5-turbo', 'anthropic:claude-2'];
+      const result = filterProviderConfigs(providers, 'openai');
+      expect(result).toEqual(['openai:gpt-4', 'openai:gpt-3.5-turbo']);
+    });
+
+    it('should filter ProviderOptions by id', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'GPT-4' },
+        { id: 'anthropic:claude-2', label: 'Claude' },
+      ];
+      const result = filterProviderConfigs(providers, 'openai');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptions[])[0].id).toBe('openai:gpt-4');
+    });
+
+    it('should filter ProviderOptions by label', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'Dev' },
+        { id: 'openai:gpt-4', label: 'Stage' },
+        { id: 'openai:gpt-4', label: 'Prod' },
+      ];
+      const result = filterProviderConfigs(providers, 'Dev');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptions[])[0].label).toBe('Dev');
+    });
+
+    it('should filter ProviderOptionsMap by key (provider id)', () => {
+      const providers: ProviderOptionsMap[] = [
+        { 'openai:gpt-4': { label: 'Dev' } },
+        { 'anthropic:claude-2': { label: 'Stage' } },
+      ];
+      const result = filterProviderConfigs(providers, 'openai');
+      expect(result).toHaveLength(1);
+    });
+
+    it('should filter ProviderOptionsMap by nested label', () => {
+      const providers: ProviderOptionsMap[] = [
+        { 'openai:gpt-4': { label: 'Dev' } },
+        { 'openai:gpt-4': { label: 'Stage' } },
+      ];
+      const result = filterProviderConfigs(providers, 'Stage');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptionsMap[])[0]['openai:gpt-4'].label).toBe('Stage');
+    });
+
+    it('should handle mixed array of provider types', () => {
+      const fn = Object.assign(async () => ({ output: '' }), { label: 'CustomDev' });
+      const providers: TestSuiteConfig['providers'] = [
+        'openai:gpt-4',
+        { id: 'anthropic:claude-2', label: 'ClaudeDev' },
+        { 'google:gemini': { label: 'GeminiDev' } } as ProviderOptionsMap,
+        fn,
+      ];
+      const result = filterProviderConfigs(providers, 'Dev');
+      expect(result).toHaveLength(3);
+    });
+
+    it('should return empty array if no providers match', () => {
+      const providers = ['openai:gpt-4', 'anthropic:claude-2'];
+      const result = filterProviderConfigs(providers, 'nonexistent');
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe('regex patterns', () => {
+    it('should support complex regex patterns', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'Dev-US' },
+        { id: 'openai:gpt-4', label: 'Dev-EU' },
+        { id: 'openai:gpt-4', label: 'Prod-US' },
+      ];
+      const result = filterProviderConfigs(providers, 'Dev-.*');
+      expect(result).toHaveLength(2);
+    });
+
+    it('should match either id or label with regex', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'Production' },
+        { id: 'anthropic:claude-sonnet', label: 'Dev' },
+      ];
+      const result = filterProviderConfigs(providers, '(gpt|sonnet)');
+      expect(result).toHaveLength(2);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle empty array', () => {
+      const result = filterProviderConfigs([], 'anything');
+      expect(result).toEqual([]);
+    });
+
+    it('should handle provider without label filtering by id only', () => {
+      const providers: ProviderOptions[] = [{ id: 'openai:gpt-4' }];
+      const result = filterProviderConfigs(providers, 'gpt-4');
+      expect(result).toHaveLength(1);
+    });
+
+    it('should handle ProviderOptionsMap with overridden id', () => {
+      const providers: ProviderOptionsMap[] = [
+        { 'file://custom.js': { id: 'my-custom-id', label: 'Dev' } },
+      ];
+      // Should match the overridden id, not the key
+      const result = filterProviderConfigs(providers, 'my-custom-id');
+      expect(result).toHaveLength(1);
+    });
+
+    it('should filter HTTP providers with config by label (bug report case)', () => {
+      const providers: ProviderOptions[] = [
+        {
+          id: 'https',
+          label: 'direct-haiku-4.5',
+          config: {
+            url: 'http://localhost:3333/assistant',
+            method: 'POST',
+            headers: {
+              Accept: '{{acceptHeader}}',
+              'Content-Type': 'application/json',
+              'X-Assistant-Model': 'anthropic:claude-haiku-4-5-20251001',
+              'X-Conversation-ID': '{{conversationId}}',
+              'X-System-ID': '{{systemId}}',
+            },
+            body: {
+              prompt: '{{prompt}}',
+            },
+          },
+        },
+        {
+          id: 'https',
+          label: 'haiku-4.5',
+          config: {
+            url: 'http://localhost:3333/assistant',
+            method: 'POST',
+            headers: {
+              Accept: '{{acceptHeader}}',
+              'Content-Type': 'application/json',
+              'X-Assistant-Model': 'openrouter:anthropic/claude-haiku-4.5',
+              'X-Conversation-ID': '{{conversationId}}',
+              'X-System-ID': '{{systemId}}',
+            },
+            body: {
+              prompt: '{{prompt}}',
+            },
+            transformResponse: 'json',
+          },
+        },
+      ];
+      const result = filterProviderConfigs(providers, 'direct-haiku-4.5');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptions[])[0].label).toBe('direct-haiku-4.5');
+    });
+
+    it('should handle provider with null or empty id', () => {
+      const providers: Array<Partial<ProviderOptions>> = [
+        { id: null as any, label: 'Provider1', config: {} },
+        { id: '', label: 'Provider2', config: {} },
+        { id: undefined, label: 'Provider3', config: {} },
+      ];
+      const result = filterProviderConfigs(providers as any, 'Provider2');
+      expect(result).toHaveLength(1);
+      expect((result as Array<Partial<ProviderOptions>>)[0].label).toBe('Provider2');
+    });
+  });
+});

test/util/eval/filterTests.test.ts

@@ -0,0 +1,797 @@
+import path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import Eval from '../../../src/models/eval';
+import { ResultFailureReason } from '../../../src/types/index';
+import { filterTests } from '../../../src/util/eval/filterTests';
+
+import type { TestCase, TestSuite } from '../../../src/types/index';
+
+vi.mock('../../../src/models/eval', () => ({
+  default: {
+    findById: vi.fn(),
+  },
+}));
+
+describe('filterTests', () => {
+  const mockTestSuite: TestSuite = {
+    prompts: [],
+    providers: [],
+    tests: [
+      {
+        description: 'test1',
+        vars: { var1: 'test1' },
+        assert: [],
+        metadata: { type: 'unit' },
+      },
+      {
+        description: 'test2',
+        vars: { var1: 'test2' },
+        assert: [],
+        metadata: { type: 'integration' },
+      },
+      {
+        description: 'test3',
+        vars: { var1: 'test3' },
+        assert: [],
+        metadata: { type: 'unit' },
+      },
+    ],
+  };
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    const mockEval = {
+      id: 'eval-123',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { var1: 'test1' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: mockTestSuite.tests![0],
+          },
+          {
+            vars: { var1: 'test2' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: mockTestSuite.tests![1],
+          },
+          {
+            vars: { var1: 'test3' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: mockTestSuite.tests![2],
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: {
+              reasoning: 0,
+              acceptedPrediction: 0,
+              rejectedPrediction: 0,
+            },
+          },
+        },
+      }),
+    };
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+  });
+
+  it('should return all tests if no options provided', async () => {
+    const result = await filterTests(mockTestSuite, {});
+    expect(result).toEqual(mockTestSuite.tests);
+  });
+
+  it('should return empty array if testSuite has no tests', async () => {
+    const result = await filterTests({ prompts: [], providers: [] }, {});
+    expect(result).toEqual([]);
+  });
+
+  describe('metadata filter', () => {
+    it('should filter tests by metadata key-value pair', async () => {
+      const result = await filterTests(mockTestSuite, { metadata: 'type=unit' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test3']);
+    });
+
+    it('should throw error if metadata filter format is invalid', async () => {
+      await expect(filterTests(mockTestSuite, { metadata: 'invalid' })).rejects.toThrow(
+        '--filter-metadata must be specified in key=value format',
+      );
+    });
+
+    it('should exclude tests without metadata', async () => {
+      const testSuite = {
+        ...mockTestSuite,
+        tests: [...mockTestSuite.tests!, { description: 'no-metadata', vars: {}, assert: [] }],
+      };
+      const result = await filterTests(testSuite, { metadata: 'type=unit' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test3']);
+    });
+
+    describe('multiple metadata filters', () => {
+      const multiMetadataTestSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [
+          {
+            description: 'test1',
+            vars: { var1: 'test1' },
+            assert: [],
+            metadata: { type: 'unit', env: 'dev', priority: 'high' },
+          },
+          {
+            description: 'test2',
+            vars: { var1: 'test2' },
+            assert: [],
+            metadata: { type: 'unit', env: 'prod', priority: 'low' },
+          },
+          {
+            description: 'test3',
+            vars: { var1: 'test3' },
+            assert: [],
+            metadata: { type: 'integration', env: 'dev', priority: 'high' },
+          },
+          {
+            description: 'test4',
+            vars: { var1: 'test4' },
+            assert: [],
+            metadata: { type: 'integration', env: 'prod', priority: 'medium' },
+          },
+        ],
+      };
+
+      it('should filter tests matching ALL metadata conditions (AND logic)', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit', 'env=dev'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+
+      it('should return empty when no tests match all conditions', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit', 'env=staging'],
+        });
+        expect(result).toHaveLength(0);
+      });
+
+      it('should handle single string value (backward compatibility)', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: 'type=unit',
+        });
+        expect(result).toHaveLength(2);
+        expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+      });
+
+      it('should handle array with single element', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit'],
+        });
+        expect(result).toHaveLength(2);
+        expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+      });
+
+      it('should filter with three or more conditions', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit', 'env=dev', 'priority=high'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+
+      it('should handle values containing equals sign', async () => {
+        const testSuiteWithEquals: TestSuite = {
+          prompts: [],
+          providers: [],
+          tests: [
+            {
+              description: 'test-with-equals',
+              vars: { var1: 'test1' },
+              assert: [],
+              metadata: { query: 'a=1&b=2', type: 'special' },
+            },
+          ],
+        };
+        const result = await filterTests(testSuiteWithEquals, {
+          metadata: ['query=a=1&b=2'],
+        });
+        expect(result).toHaveLength(1);
+      });
+
+      it('should throw error if any filter in array is invalid', async () => {
+        await expect(
+          filterTests(multiMetadataTestSuite, {
+            metadata: ['type=unit', 'invalid'],
+          }),
+        ).rejects.toThrow('--filter-metadata must be specified in key=value format');
+      });
+
+      it('should throw error for empty value', async () => {
+        await expect(
+          filterTests(multiMetadataTestSuite, {
+            metadata: ['type='],
+          }),
+        ).rejects.toThrow('--filter-metadata must be specified in key=value format');
+      });
+
+      it('should exclude tests missing any of the required metadata keys', async () => {
+        const testSuitePartialMetadata: TestSuite = {
+          prompts: [],
+          providers: [],
+          tests: [
+            {
+              description: 'has-both',
+              vars: { var1: 'test1' },
+              assert: [],
+              metadata: { type: 'unit', env: 'dev' },
+            },
+            {
+              description: 'missing-env',
+              vars: { var1: 'test2' },
+              assert: [],
+              metadata: { type: 'unit' },
+            },
+          ],
+        };
+        const result = await filterTests(testSuitePartialMetadata, {
+          metadata: ['type=unit', 'env=dev'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+
+      it('should work with array metadata values', async () => {
+        const testSuiteArrayMetadata: TestSuite = {
+          prompts: [],
+          providers: [],
+          tests: [
+            {
+              description: 'has-security-tag',
+              vars: { var1: 'test1' },
+              assert: [],
+              metadata: { type: 'unit', tags: ['security', 'auth'] },
+            },
+            {
+              description: 'no-security-tag',
+              vars: { var1: 'test2' },
+              assert: [],
+              metadata: { type: 'unit', tags: ['performance'] },
+            },
+          ],
+        };
+        const result = await filterTests(testSuiteArrayMetadata, {
+          metadata: ['type=unit', 'tags=security'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+    });
+  });
+
+  describe('failing filter', () => {
+    it('should filter failing tests when failing option is provided', async () => {
+      // --filter-failing returns all non-successful results (failures + errors)
+      const result = await filterTests(mockTestSuite, { failing: 'eval-123' });
+      expect(result).toHaveLength(3);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2', 'test3']);
+    });
+
+    it('should match failing tests when provider paths differ', async () => {
+      vi.resetAllMocks();
+      const absPath = path.join(process.cwd(), 'provider.js');
+      const mockEval = {
+        id: 'eval-123',
+        createdAt: new Date().getTime(),
+        config: {},
+        results: [],
+        resultsCount: 0,
+        prompts: [],
+        persisted: true,
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          version: 2,
+          timestamp: new Date().toISOString(),
+          results: [
+            {
+              vars: { var1: 'test1' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              provider: { id: `file://${absPath}` },
+              testCase: mockTestSuite.tests![0],
+            },
+          ],
+          table: { head: { prompts: [], vars: [] }, body: [] },
+          stats: {
+            successes: 0,
+            failures: 0,
+            errors: 0,
+            tokenUsage: {
+              total: 0,
+              prompt: 0,
+              completion: 0,
+              cached: 0,
+              numRequests: 0,
+              completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+            },
+          },
+        }),
+      };
+
+      vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+      const testSuite = {
+        ...mockTestSuite,
+        tests: [
+          {
+            ...mockTestSuite.tests![0],
+            provider: `file://./provider.js`,
+          },
+        ],
+      } as TestSuite;
+
+      const result = await filterTests(testSuite, { failing: 'eval-123' });
+      expect(result).toHaveLength(1);
+    });
+  });
+
+  describe('errors only filter', () => {
+    it('should filter error tests when errorsOnly option is provided', async () => {
+      const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-123' });
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test2');
+    });
+  });
+
+  describe('failing only filter', () => {
+    it('should filter assertion failures only, excluding errors', async () => {
+      // --filter-failing-only returns only assertion failures, not errors
+      const result = await filterTests(mockTestSuite, { failingOnly: 'eval-123' });
+      expect(result).toHaveLength(2);
+      // test1 and test3 have ASSERT failures, test2 has ERROR
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test3']);
+    });
+
+    it('should return empty when all failures are errors', async () => {
+      vi.resetAllMocks();
+      const mockEval = {
+        id: 'eval-456',
+        createdAt: new Date().getTime(),
+        config: {},
+        results: [],
+        resultsCount: 0,
+        prompts: [],
+        persisted: true,
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          version: 2,
+          timestamp: new Date().toISOString(),
+          results: [
+            {
+              vars: { var1: 'test1' },
+              success: false,
+              failureReason: ResultFailureReason.ERROR,
+              testCase: mockTestSuite.tests![0],
+            },
+            {
+              vars: { var1: 'test2' },
+              success: false,
+              failureReason: ResultFailureReason.ERROR,
+              testCase: mockTestSuite.tests![1],
+            },
+          ],
+          table: { head: { prompts: [], vars: [] }, body: [] },
+          stats: {
+            successes: 0,
+            failures: 0,
+            errors: 2,
+            tokenUsage: {
+              total: 0,
+              prompt: 0,
+              completion: 0,
+              cached: 0,
+              numRequests: 0,
+              completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+            },
+          },
+        }),
+      };
+      vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+      const result = await filterTests(mockTestSuite, { failingOnly: 'eval-456' });
+      expect(result).toHaveLength(0);
+    });
+
+    it('should combine failingOnly and errorsOnly when both provided', async () => {
+      // When both are provided, it should be a union of assertion failures and errors
+      const result = await filterTests(mockTestSuite, {
+        failingOnly: 'eval-123',
+        errorsOnly: 'eval-123',
+      });
+      // Should include all 3 tests: test1 (ASSERT), test2 (ERROR), test3 (ASSERT)
+      expect(result).toHaveLength(3);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(
+        expect.arrayContaining(['test1', 'test2', 'test3']),
+      );
+    });
+  });
+
+  describe('pattern filter', () => {
+    it('should filter tests by description pattern', async () => {
+      const result = await filterTests(mockTestSuite, { pattern: 'test[12]' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+    });
+
+    it('should handle tests without description', async () => {
+      const testSuite = {
+        ...mockTestSuite,
+        tests: [...mockTestSuite.tests!, { vars: {}, assert: [] }],
+      };
+      const result = await filterTests(testSuite, { pattern: 'test' });
+      expect(result).toHaveLength(3);
+    });
+
+    it('should throw error for invalid regex pattern', async () => {
+      await expect(filterTests(mockTestSuite, { pattern: '[invalid' })).rejects.toThrow(
+        /Invalid regex pattern "\[invalid"/,
+      );
+    });
+  });
+
+  describe('range filter', () => {
+    it('should slice tests by zero-based start-inclusive, end-exclusive range', async () => {
+      const result = await filterTests(mockTestSuite, { range: '1:3' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test2', 'test3']);
+    });
+
+    it('should support omitted start and omitted end', async () => {
+      const fromStart = await filterTests(mockTestSuite, { range: ':2' });
+      expect(fromStart.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+
+      const toEnd = await filterTests(mockTestSuite, { range: '1:' });
+      expect(toEnd.map((t: TestCase) => t.vars?.var1)).toEqual(['test2', 'test3']);
+    });
+
+    it('should allow range ends beyond the test count', async () => {
+      const result = await filterTests(mockTestSuite, { range: '2:100' });
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test3');
+    });
+
+    it('should apply after pattern and before firstN', async () => {
+      const result = await filterTests(mockTestSuite, {
+        pattern: 'test',
+        range: '1:3',
+        firstN: 1,
+      });
+
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test2');
+    });
+
+    it('should throw error for invalid range format', async () => {
+      await expect(filterTests(mockTestSuite, { range: '1' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+      await expect(filterTests(mockTestSuite, { range: ':' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+      await expect(filterTests(mockTestSuite, { range: 'a:b' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+    });
+
+    it('should reject invalid whitespace-heavy ranges without excessive backtracking', async () => {
+      await expect(
+        filterTests(mockTestSuite, { range: `${'\t'.repeat(10_000)}:${'\t'.repeat(10_000)}x` }),
+      ).rejects.toThrow(/--filter-range must be specified in start:end format/);
+    });
+
+    it('should throw error when start is greater than end', async () => {
+      await expect(filterTests(mockTestSuite, { range: '3:1' })).rejects.toThrow(
+        '--filter-range start must be less than or equal to end, got: 3:1',
+      );
+    });
+
+    it('should reject negative range bounds', async () => {
+      await expect(filterTests(mockTestSuite, { range: '-1:5' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+      await expect(filterTests(mockTestSuite, { range: '0:-3' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+    });
+
+    it('should warn when range slices to an empty set on a non-empty suite', async () => {
+      const warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => undefined as any);
+      try {
+        const result = await filterTests(mockTestSuite, { range: '100:200' });
+        expect(result).toHaveLength(0);
+        expect(warnSpy).toHaveBeenCalledWith(
+          expect.stringContaining('--filter-range 100:200 selected 0 tests'),
+        );
+      } finally {
+        warnSpy.mockRestore();
+      }
+    });
+  });
+
+  describe('firstN filter', () => {
+    it('should take first N tests', async () => {
+      const result = await filterTests(mockTestSuite, { firstN: 2 });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+    });
+
+    it('should handle string input for firstN', async () => {
+      const result = await filterTests(mockTestSuite, { firstN: '2' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+    });
+
+    it('should throw error if firstN is not a number', async () => {
+      await expect(filterTests(mockTestSuite, { firstN: 'invalid' })).rejects.toThrow(
+        'firstN must be a number, got: invalid',
+      );
+    });
+
+    it('should handle undefined firstN', async () => {
+      const result = await filterTests(mockTestSuite, { firstN: undefined });
+      expect(result).toEqual(mockTestSuite.tests);
+    });
+
+    it('should throw error if firstN is null', async () => {
+      await expect(filterTests(mockTestSuite, { firstN: null as any })).rejects.toThrow(
+        'firstN must be a number, got: null',
+      );
+    });
+
+    it('should throw error if firstN is NaN', async () => {
+      await expect(filterTests(mockTestSuite, { firstN: Number.NaN })).rejects.toThrow(
+        'firstN must be a number, got: NaN',
+      );
+    });
+  });
+
+  describe('sample filter', () => {
+    it('should take N random tests', async () => {
+      const result = await filterTests(mockTestSuite, { sample: 2 });
+      expect(result).toHaveLength(2);
+      // Can't test exact values since it's random
+      expect(result.every((t: TestCase) => mockTestSuite.tests!.includes(t))).toBe(true);
+    });
+
+    it('should handle string input for sample', async () => {
+      const result = await filterTests(mockTestSuite, { sample: '2' });
+      expect(result).toHaveLength(2);
+      expect(result.every((t: TestCase) => mockTestSuite.tests!.includes(t))).toBe(true);
+    });
+
+    it('should return the same sample when given the same seed', async () => {
+      const randomSpy = vi.spyOn(Math, 'random').mockImplementation(() => {
+        throw new Error('seeded sampling should not use Math.random');
+      });
+      try {
+        const first = await filterTests(mockTestSuite, { sample: 2, sampleSeed: 42 });
+        const second = await filterTests(mockTestSuite, { sample: 2, sampleSeed: 42 });
+
+        expect(first.map((test) => test.vars?.var1)).toEqual(second.map((test) => test.vars?.var1));
+        expect(randomSpy).not.toHaveBeenCalled();
+      } finally {
+        randomSpy.mockRestore();
+      }
+    });
+
+    it('should throw error if sample is not a number', async () => {
+      await expect(filterTests(mockTestSuite, { sample: 'invalid' })).rejects.toThrow(
+        'sample must be a number, got: invalid',
+      );
+    });
+
+    it('should handle undefined sample', async () => {
+      const result = await filterTests(mockTestSuite, { sample: undefined });
+      expect(result).toEqual(mockTestSuite.tests);
+    });
+
+    it('should throw error if sample is null', async () => {
+      await expect(filterTests(mockTestSuite, { sample: null as any })).rejects.toThrow(
+        'sample must be a number, got: null',
+      );
+    });
+
+    it('should throw error if sample is NaN', async () => {
+      await expect(filterTests(mockTestSuite, { sample: Number.NaN })).rejects.toThrow(
+        'sample must be a number, got: NaN',
+      );
+    });
+  });
+
+  describe('multiple filters', () => {
+    it('should compose metadata and failing filters', async () => {
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: { var1: 'test1' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: mockTestSuite.tests![0],
+            },
+            {
+              vars: { var1: 'test2' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: mockTestSuite.tests![1],
+            },
+            {
+              vars: { var1: 'test3' },
+              success: true,
+              testCase: mockTestSuite.tests![2],
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(mockTestSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test1');
+    });
+
+    it('should not reintroduce metadata-excluded tests with matching vars', async () => {
+      const excludedTest: TestCase = {
+        description: 'integration test',
+        vars: { prompt: 'shared input' },
+        metadata: { type: 'integration' },
+        provider: 'provider-integration',
+      };
+      const selectedTest: TestCase = {
+        description: 'unit test',
+        vars: { prompt: 'shared input' },
+        metadata: { type: 'unit' },
+        provider: 'provider-unit',
+      };
+      const testSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [excludedTest, selectedTest],
+      };
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: excludedTest.vars,
+              provider: { id: 'provider-integration' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: excludedTest,
+            },
+            {
+              vars: selectedTest.vars,
+              provider: { id: 'provider-unit' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: selectedTest,
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(testSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result).toEqual([selectedTest]);
+    });
+
+    it('should preserve generated result tests when metadata matches all config tests', async () => {
+      const configTest: TestCase = {
+        description: 'configured test',
+        vars: { prompt: 'configured' },
+        metadata: { type: 'unit' },
+      };
+      const generatedTest: TestCase = {
+        description: 'generated test',
+        vars: { prompt: 'generated' },
+        metadata: { type: 'unit', pluginId: 'cipher-code' },
+      };
+      const testSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [configTest],
+      };
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: configTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: configTest,
+            },
+            {
+              vars: generatedTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: generatedTest,
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(testSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result.map((test) => test.description)).toEqual(['configured test', 'generated test']);
+    });
+
+    it('should filter generated result tests by metadata before deduplication', async () => {
+      const excludedGeneratedTest: TestCase = {
+        description: 'excluded generated test',
+        vars: { prompt: 'shared generated input' },
+        metadata: { type: 'integration' },
+      };
+      const selectedGeneratedTest: TestCase = {
+        description: 'selected generated test',
+        vars: { prompt: 'shared generated input' },
+        metadata: { type: 'unit' },
+      };
+      const testSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [],
+      };
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: excludedGeneratedTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: excludedGeneratedTest,
+            },
+            {
+              vars: selectedGeneratedTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: selectedGeneratedTest,
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(testSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result).toEqual([selectedGeneratedTest]);
+    });
+  });
+});

test/util/eval/redteamWarning.test.ts

@@ -0,0 +1,63 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import { warnIfRedteamConfigHasNoTests } from '../../../src/util/eval/redteamWarning';
+
+import type { TestSuite, UnifiedConfig } from '../../../src/types/index';
+
+vi.mock('../../../src/logger', () => {
+  const mockLogger = {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  };
+
+  return {
+    __esModule: true,
+    default: mockLogger,
+  };
+});
+
+describe('redteam warning in eval command', () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should warn when config has redteam section but no test cases', () => {
+    const config = {
+      redteam: {
+        purpose: 'Test red team purpose',
+      },
+    } as Partial<UnifiedConfig>;
+    const testSuite = {
+      prompts: [],
+      providers: [],
+      tests: [],
+    } as TestSuite;
+
+    warnIfRedteamConfigHasNoTests(config, testSuite);
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining('redteam section but no test cases'),
+    );
+    expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('promptfoo redteam generate'));
+  });
+
+  it('should not warn when scenarios are present even if tests is empty', () => {
+    const config = {
+      redteam: {
+        purpose: 'Test red team purpose',
+      },
+    } as Partial<UnifiedConfig>;
+    const testSuite = {
+      prompts: [],
+      providers: [],
+      tests: [],
+      scenarios: [{ config: [], tests: [] }],
+    } as TestSuite;
+
+    warnIfRedteamConfigHasNoTests(config, testSuite);
+
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+});

test/util/eval/summary.test.ts

@@ -0,0 +1,891 @@
+import chalk from 'chalk';
+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
+import { generateEvalSummary } from '../../../src/util/eval/summary';
+import { stripAnsi } from '../../util/utils';
+
+import type { EvalSummaryParams } from '../../../src/util/eval/summary';
+import type { TokenUsageTracker } from '../../../src/util/tokenUsage';
+
+type MockTracker = {
+  getProviderIds: Mock;
+  getProviderUsage: Mock;
+  trackUsage: Mock;
+  resetAllUsage: Mock;
+  resetProviderUsage: Mock;
+  getTotalUsage: Mock;
+  cleanup: Mock;
+};
+
+function createMockTracker(): TokenUsageTracker {
+  return {
+    getProviderIds: vi.fn().mockReturnValue([]),
+    getProviderUsage: vi.fn(),
+    trackUsage: vi.fn(),
+    resetAllUsage: vi.fn(),
+    resetProviderUsage: vi.fn(),
+    getTotalUsage: vi.fn(),
+    cleanup: vi.fn(),
+  } as unknown as TokenUsageTracker;
+}
+
+describe('generateEvalSummary', () => {
+  let mockTracker: MockTracker & TokenUsageTracker;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockTracker = createMockTracker() as MockTracker & TokenUsageTracker;
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('completion message', () => {
+    it('should show basic completion message when not writing to database', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-123',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Eval complete');
+      expect(output).not.toContain('eval-123');
+    });
+
+    it('should show eval ID when writing to database without shareable URL', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-456',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Eval complete (ID: eval-456)');
+    });
+
+    it('should show shareable URL when available', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-789',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: 'https://promptfoo.app/eval/abc123',
+        wantsToShare: true,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Eval complete: https://promptfoo.app/eval/abc123');
+      expect(output).not.toContain('eval-789');
+    });
+
+    it('should say "Red team complete" for red team evals', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-rt-1',
+        isRedteam: true,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 10,
+        failures: 2,
+        errors: 0,
+        duration: 8000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Red team complete');
+      expect(output).not.toContain('Eval complete');
+    });
+
+    it('should explain non-retryable target errors when an eval is aborted', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-aborted',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 0,
+        failures: 0,
+        errors: 1,
+        duration: 1000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+        targetErrorStatus: 401,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✗ Eval aborted (ID: eval-aborted)');
+      expect(output).toContain(
+        'Scan stopped: Target is unavailable and will not recover on retry.',
+      );
+      expect(output).toContain('Target returned HTTP 401');
+      expect(output).not.toContain('Server error (500)');
+      expect(output).toContain('To fix: Check your target configuration and credentials.');
+    });
+  });
+
+  describe('token usage', () => {
+    it('should display eval tokens correctly', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-123',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          prompt: 400,
+          completion: 600,
+          cached: 0,
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion)');
+    });
+
+    it('should display grading tokens only when no eval tokens (critical bug fix)', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-grading-only',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 0,
+          assertions: {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+            cached: 0,
+          },
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Grading: 500 (200 prompt, 300 completion)');
+      expect(output).not.toContain('Eval:');
+    });
+
+    it('should display both eval and grading tokens', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-both',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          prompt: 400,
+          completion: 600,
+          assertions: {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+          },
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion)');
+      expect(output).toContain('Grading: 500 (200 prompt, 300 completion)');
+    });
+
+    it('should show 100% cached correctly', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-cached',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          cached: 1000,
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Eval: 1,000 (cached)');
+    });
+
+    it('should show partial cached tokens', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-partial-cache',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          prompt: 400,
+          completion: 600,
+          cached: 200,
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion, 200 cached)');
+    });
+
+    it('should not show token section when no tokens', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-no-tokens',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).not.toContain('Tokens:');
+    });
+  });
+
+  describe('provider breakdown', () => {
+    it('should show provider breakdown with request counts', () => {
+      mockTracker.getProviderIds.mockReturnValue(['openai:gpt-4', 'anthropic:claude-3']);
+      mockTracker.getProviderUsage.mockImplementation((id: string) => {
+        if (id === 'openai:gpt-4') {
+          return {
+            total: 1500,
+            prompt: 600,
+            completion: 900,
+            cached: 0,
+            numRequests: 5,
+          };
+        }
+        if (id === 'anthropic:claude-3') {
+          return {
+            total: 800,
+            prompt: 300,
+            completion: 500,
+            cached: 0,
+            numRequests: 3,
+          };
+        }
+        return undefined;
+      });
+
+      const params: EvalSummaryParams = {
+        evalId: 'eval-providers',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 2300 },
+        successes: 8,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Providers:');
+      expect(output).toContain('openai:gpt-4');
+      expect(output).toContain('1,500 (5 requests; 600 prompt, 900 completion)');
+      expect(output).toContain('anthropic:claude-3');
+      expect(output).toContain('800 (3 requests; 300 prompt, 500 completion)');
+    });
+
+    it('should always show request count even when 0', () => {
+      mockTracker.getProviderIds.mockReturnValue(['openai:gpt-4', 'anthropic:claude-3']);
+      mockTracker.getProviderUsage.mockImplementation((id: string) => {
+        if (id === 'openai:gpt-4') {
+          return {
+            total: 1000,
+            cached: 1000,
+            numRequests: 0,
+          };
+        }
+        if (id === 'anthropic:claude-3') {
+          return {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+            numRequests: 2,
+          };
+        }
+        return undefined;
+      });
+
+      const params: EvalSummaryParams = {
+        evalId: 'eval-zero-requests',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 1500 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Providers:');
+      expect(output).toContain('openai:gpt-4: 1,000 (0 requests; cached)');
+      expect(output).toContain('anthropic:claude-3: 500 (2 requests; 200 prompt, 300 completion)');
+    });
+  });
+
+  describe('pass rate and results', () => {
+    it('should show percentages for each result line at 100% pass rate', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-100',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 10,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('Results:');
+      expect(plainOutput).toContain('10 passed (100%)');
+      expect(plainOutput).toContain('0 failed (0%)');
+      expect(plainOutput).toContain('0 errors (0%)');
+    });
+
+    it('should show percentages for each result line when some tests fail', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-85',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 17,
+        failures: 3,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('Results:');
+      expect(plainOutput).toContain('17 passed (85.00%)');
+      expect(plainOutput).toContain('3 failed (15.00%)');
+      expect(plainOutput).toContain('0 errors (0%)');
+    });
+
+    it('should show percentages for each result line when passed and failed are split evenly', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-50',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 5,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('Results:');
+      expect(plainOutput).toContain('5 passed (50.00%)');
+      expect(plainOutput).toContain('5 failed (50.00%)');
+      expect(plainOutput).toContain('0 errors (0%)');
+    });
+
+    it('should include errors in results', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-errors',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 8,
+        failures: 1,
+        errors: 1,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+      const outputLines = plainOutput.split('\n');
+      const hasLineMatching = (pattern: RegExp) => outputLines.some((line) => pattern.test(line));
+
+      expect(plainOutput).toContain('Results:');
+      expect(hasLineMatching(/^\s*(✓\s+)?8 passed \(80\.00%\)$/)).toBe(true);
+      expect(hasLineMatching(/^\s*(✗\s+)?1 failed \(10\.00%\)$/)).toBe(true);
+      expect(hasLineMatching(/^\s*(✗\s+)?1 error \(10\.00%\)$/)).toBe(true);
+      expect(plainOutput).not.toContain('1 errors');
+    });
+
+    it('should render colored icons with muted percentages', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-styling',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 8,
+        failures: 1,
+        errors: 1,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+
+      expect(lines).toContain(
+        `  ${chalk.green('✓')} ${chalk.white.bold('8')} ${chalk.white('passed')} ${chalk.gray('(80.00%)')}`,
+      );
+      expect(lines).toContain(
+        `  ${chalk.red('✗')} ${chalk.white.bold('1')} ${chalk.white('failed')} ${chalk.gray('(10.00%)')}`,
+      );
+      expect(lines).toContain(
+        `  ${chalk.red('✗')} ${chalk.white.bold('1')} ${chalk.white('error')} ${chalk.gray('(10.00%)')}`,
+      );
+    });
+  });
+
+  describe('guidance messages', () => {
+    it('should show guidance when writing to database without shareable URL', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-view',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('» View results: promptfoo view');
+      expect(output).toContain('» Share with your team: https://promptfoo.app');
+      expect(output).toContain('» Feedback: https://promptfoo.dev/feedback');
+    });
+
+    it('should show share guidance with cloud enabled', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-share-cloud',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: true,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('» View results: promptfoo view');
+      expect(output).toContain('» Create shareable URL: promptfoo share');
+      expect(output).not.toContain('https://promptfoo.app');
+    });
+
+    it('should NOT show share guidance when explicitly disabled (--no-share)', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-no-share',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: true,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('» View results: promptfoo view');
+      expect(output).not.toContain('» Share with your team');
+      expect(output).not.toContain('» Create shareable URL');
+    });
+
+    it('should NOT show guidance when not writing to database', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-no-write',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).not.toContain('» View results:');
+      expect(output).not.toContain('» Share');
+      expect(output).not.toContain('» Feedback:');
+    });
+
+    it('should NOT show guidance when shareable URL is present', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-with-url',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: 'https://promptfoo.app/eval/abc123',
+        wantsToShare: true,
+        hasExplicitDisable: false,
+        cloudEnabled: true,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).not.toContain('» View results:');
+      expect(output).not.toContain('» Share');
+      expect(output).not.toContain('» Feedback:');
+    });
+  });
+
+  describe('performance metrics', () => {
+    it('should show duration and concurrency', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-perf',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 125, // 125 seconds = 2m 5s
+        maxConcurrency: 8,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Duration:');
+      expect(output).toContain('(concurrency: 8)');
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should use singular "error" when there is exactly 1 error', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-singular-error',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 1,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('1 error');
+      expect(plainOutput).not.toContain('1 errors');
+    });
+
+    it('should use plural "errors" when there are multiple errors', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-plural-errors',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 3,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('3 errors');
+    });
+
+    it('should use plural "errors" when there are 0 errors', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-zero-errors',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('0 errors');
+    });
+
+    it('should handle provider returning undefined usage gracefully', () => {
+      mockTracker.getProviderIds.mockReturnValue([
+        'openai:gpt-4',
+        'missing-provider',
+        'anthropic:claude-3',
+      ]);
+      mockTracker.getProviderUsage.mockImplementation((id: string) => {
+        if (id === 'openai:gpt-4') {
+          return {
+            total: 1000,
+            prompt: 400,
+            completion: 600,
+            numRequests: 5,
+          };
+        }
+        if (id === 'missing-provider') {
+          return undefined; // Simulates a provider that returns undefined
+        }
+        if (id === 'anthropic:claude-3') {
+          return {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+            numRequests: 3,
+          };
+        }
+        return undefined;
+      });
+
+      const params: EvalSummaryParams = {
+        evalId: 'eval-undefined-provider',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 1500 },
+        successes: 8,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      // Should not throw
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      // Should show the providers that have valid usage
+      expect(output).toContain('Providers:');
+      expect(output).toContain('openai:gpt-4');
+      expect(output).toContain('anthropic:claude-3');
+      // Should NOT show the missing provider
+      expect(output).not.toContain('missing-provider');
+    });
+  });
+});

test/util/exportToFile/convertTestResultsToTableRow.test.ts

@@ -0,0 +1,419 @@
+import { describe, expect, it } from 'vitest';
+import { convertTestResultsToTableRow } from '../../../src/util/exportToFile/index';
+
+import type EvalResult from '../../../src/models/evalResult';
+
+/**
+ * Creates a minimal mock EvalResult for testing.
+ * We use a partial type and cast to avoid needing the full EvalResult class.
+ */
+function createMockEvalResult(overrides: Partial<EvalResult> = {}): EvalResult {
+  const defaultResult = {
+    id: 'test-id',
+    testIdx: 0,
+    promptIdx: 0,
+    description: 'Test description',
+    testCase: {
+      vars: {},
+    },
+    prompt: {
+      raw: 'test prompt',
+    },
+    response: {
+      output: 'test output',
+    },
+    provider: {
+      id: 'test-provider',
+      label: 'Test Provider',
+    },
+    success: true,
+    score: 1,
+    latencyMs: 100,
+    error: undefined,
+    metadata: undefined,
+    namedScores: {},
+    gradingResult: undefined,
+    cost: 0,
+  };
+
+  return {
+    ...defaultResult,
+    ...overrides,
+    testCase: {
+      ...defaultResult.testCase,
+      ...(overrides.testCase || {}),
+    },
+    prompt: {
+      ...defaultResult.prompt,
+      ...(overrides.prompt || {}),
+    },
+    response: {
+      ...defaultResult.response,
+      ...(overrides.response || {}),
+    },
+    provider: {
+      ...defaultResult.provider,
+      ...(overrides.provider || {}),
+    },
+  } as EvalResult;
+}
+
+describe('convertTestResultsToTableRow', () => {
+  describe('basic functionality', () => {
+    it('should convert results to table row format', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { question: 'What is AI?' } },
+        }),
+      ];
+      const varsForHeader = ['question'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.description).toBe('Test description');
+      expect(row.vars).toEqual(['What is AI?']);
+      expect(row.testIdx).toBe(0);
+      expect(row.outputs).toHaveLength(1);
+      expect(row.outputs[0].text).toBe('test output');
+    });
+
+    it('should handle multiple variables', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: {
+            vars: {
+              question: 'What is AI?',
+              context: 'Some context',
+              temperature: 0.5,
+            },
+          },
+        }),
+      ];
+      const varsForHeader = ['context', 'question', 'temperature'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['Some context', 'What is AI?', '0.5']);
+    });
+
+    it('should handle missing variables', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { question: 'What is AI?' } },
+        }),
+      ];
+      const varsForHeader = ['question', 'missingVar'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['What is AI?', '']);
+    });
+
+    it('should handle object variables as JSON', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: {
+            vars: {
+              config: { nested: 'value', count: 42 },
+            },
+          },
+        }),
+      ];
+      const varsForHeader = ['config'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars[0]).toBe('{"nested":"value","count":42}');
+    });
+  });
+
+  describe('sessionId handling', () => {
+    it('should use sessionId from testCase.vars when present', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { sessionId: 'user-session-123' } },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['user-session-123']);
+    });
+
+    it('should use metadata.sessionId when vars.sessionId is not present', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: { sessionId: 'metadata-session-456' },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['metadata-session-456']);
+    });
+
+    it('should use metadata.sessionId when vars.sessionId is empty string', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { sessionId: '' } },
+          metadata: { sessionId: 'metadata-session-789' },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['metadata-session-789']);
+    });
+
+    it('should not overwrite non-empty vars.sessionId with metadata.sessionId', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { sessionId: 'user-provided-session' } },
+          metadata: { sessionId: 'should-be-ignored' },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['user-provided-session']);
+    });
+
+    it('should return empty string when no sessionId is available', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: {},
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['']);
+    });
+  });
+
+  describe('sessionIds array handling (multi-turn strategies)', () => {
+    it('should join sessionIds array with newlines', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: { sessionIds: ['session-1', 'session-2', 'session-3'] },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['session-1\nsession-2\nsession-3']);
+    });
+
+    it('should handle single-element sessionIds array', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: { sessionIds: ['only-session'] },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['only-session']);
+    });
+
+    it('should prefer sessionIds array over sessionId when both exist', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: {
+            sessionId: 'single-session',
+            sessionIds: ['multi-1', 'multi-2'],
+          },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['multi-1\nmulti-2']);
+    });
+
+    it('should fall back to sessionId when sessionIds is empty array', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: {
+            sessionId: 'fallback-session',
+            sessionIds: [],
+          },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['fallback-session']);
+    });
+
+    it('should ignore non-array sessionIds and use sessionId instead', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: {
+            sessionId: 'fallback-session',
+            sessionIds: 'not-an-array' as unknown as string[],
+          },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['fallback-session']);
+    });
+
+    it('should not use sessionIds array if vars.sessionId is provided', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { sessionId: 'user-provided' } },
+          metadata: { sessionIds: ['should-be-ignored-1', 'should-be-ignored-2'] },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['user-provided']);
+    });
+  });
+
+  describe('combined scenarios', () => {
+    it('should handle sessionId with other variables', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: {
+            vars: {
+              question: 'What is AI?',
+              context: 'Some context',
+            },
+          },
+          metadata: { sessionIds: ['session-a', 'session-b'] },
+        }),
+      ];
+      const varsForHeader = ['context', 'question', 'sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['Some context', 'What is AI?', 'session-a\nsession-b']);
+    });
+
+    it('should handle edge case with undefined metadata', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { question: 'What is AI?' } },
+          metadata: undefined,
+        }),
+      ];
+      const varsForHeader = ['question', 'sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.vars).toEqual(['What is AI?', '']);
+    });
+
+    it('should handle non-string sessionId values in metadata', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: {} },
+          metadata: { sessionId: { complex: 'object' } as unknown as string },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      // Should be JSON stringified
+      expect(row.vars[0]).toBe('{"complex":"object"}');
+    });
+
+    it('should handle non-string sessionId values in vars', () => {
+      const results = [
+        createMockEvalResult({
+          testCase: { vars: { sessionId: { complex: 'object' } as unknown as string } },
+        }),
+      ];
+      const varsForHeader = ['sessionId'];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      // Should be JSON stringified
+      expect(row.vars[0]).toBe('{"complex":"object"}');
+    });
+  });
+
+  describe('output formatting', () => {
+    it('should format successful output correctly', () => {
+      const results = [
+        createMockEvalResult({
+          success: true,
+          response: { output: 'The answer is 42' },
+        }),
+      ];
+      const varsForHeader: string[] = [];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.outputs[0].text).toBe('The answer is 42');
+      expect(row.outputs[0].pass).toBe(true);
+    });
+
+    it('should format error output correctly', () => {
+      const results = [
+        createMockEvalResult({
+          success: false,
+          error: 'Provider error occurred',
+          response: { output: '' },
+        }),
+      ];
+      const varsForHeader: string[] = [];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.outputs[0].text).toBe('Provider error occurred');
+      expect(row.outputs[0].pass).toBe(false);
+    });
+
+    it('should handle null output', () => {
+      const results = [
+        createMockEvalResult({
+          response: { output: null },
+          error: 'Null response',
+        }),
+      ];
+      const varsForHeader: string[] = [];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.outputs[0].text).toBe('Null response');
+    });
+
+    it('should handle object output', () => {
+      const results = [
+        createMockEvalResult({
+          response: { output: { key: 'value', nested: { data: 123 } } },
+        }),
+      ];
+      const varsForHeader: string[] = [];
+
+      const row = convertTestResultsToTableRow(results, varsForHeader);
+
+      expect(row.outputs[0].text).toBe('{"key":"value","nested":{"data":123}}');
+    });
+  });
+});

test/util/exportToFile/getHeaderForTable.test.ts

@@ -0,0 +1,178 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import { getHeaderForTable } from '../../../src/util/exportToFile/getHeaderForTable';
+
+import type Eval from '../../../src/models/eval';
+import type { TestCase } from '../../../src/types/index';
+
+describe('getHeaderForTable', () => {
+  let mockEval: Eval;
+
+  beforeEach(() => {
+    mockEval = {
+      config: {
+        defaultTest: undefined,
+      },
+      results: [],
+    } as any;
+  });
+
+  it('should handle object defaultTest with vars', () => {
+    mockEval.config.defaultTest = {
+      vars: {
+        defaultVar1: 'value1',
+        defaultVar2: 'value2',
+      },
+    };
+
+    mockEval.results = [
+      {
+        testCase: {
+          vars: {
+            testVar1: 'test1',
+            defaultVar1: 'override1',
+          },
+        } as TestCase,
+      },
+      {
+        testCase: {
+          vars: {
+            testVar2: 'test2',
+            testVar3: 'test3',
+          },
+        } as TestCase,
+      },
+    ] as any;
+
+    const header = getHeaderForTable(mockEval);
+
+    // Should include all unique var names from defaultTest and test cases
+    expect(header.vars).toContain('defaultVar1');
+    expect(header.vars).toContain('defaultVar2');
+    expect(header.vars).toContain('testVar1');
+    expect(header.vars).toContain('testVar2');
+    expect(header.vars).toContain('testVar3');
+  });
+
+  it('should handle string defaultTest', () => {
+    mockEval.config.defaultTest = 'file://path/to/defaultTest.yaml';
+
+    mockEval.results = [
+      {
+        testCase: {
+          vars: {
+            var1: 'value1',
+            var2: 'value2',
+          },
+        } as TestCase,
+      },
+    ] as any;
+
+    const header = getHeaderForTable(mockEval);
+
+    // Should only include vars from test cases when defaultTest is a string
+    expect(header.vars).toContain('var1');
+    expect(header.vars).toContain('var2');
+  });
+
+  it('should handle undefined defaultTest', () => {
+    mockEval.config.defaultTest = undefined;
+
+    mockEval.results = [
+      {
+        testCase: {
+          vars: {
+            var1: 'value1',
+            var2: 'value2',
+          },
+        } as TestCase,
+      },
+    ] as any;
+
+    const header = getHeaderForTable(mockEval);
+
+    expect(header.vars).toContain('var1');
+    expect(header.vars).toContain('var2');
+  });
+
+  it('should handle empty vars in defaultTest', () => {
+    mockEval.config.defaultTest = {
+      vars: {},
+      assert: [{ type: 'equals', value: 'test' }],
+    };
+
+    mockEval.results = [
+      {
+        testCase: {
+          vars: {
+            var1: 'value1',
+          },
+        } as TestCase,
+      },
+    ] as any;
+
+    const header = getHeaderForTable(mockEval);
+
+    expect(header.vars).toContain('var1');
+  });
+
+  it('should handle defaultTest with no vars property', () => {
+    mockEval.config.defaultTest = {
+      assert: [{ type: 'equals', value: 'test' }],
+      options: { provider: 'openai:gpt-4' },
+    };
+
+    mockEval.results = [
+      {
+        testCase: {
+          vars: {
+            var1: 'value1',
+          },
+        } as TestCase,
+      },
+    ] as any;
+
+    const header = getHeaderForTable(mockEval);
+
+    expect(header.vars).toContain('var1');
+  });
+
+  it('should deduplicate var names', () => {
+    mockEval.config.defaultTest = {
+      vars: {
+        commonVar: 'default',
+        defaultOnly: 'value',
+      },
+    };
+
+    mockEval.results = [
+      {
+        testCase: {
+          vars: {
+            commonVar: 'override1',
+            test1Var: 'value1',
+          },
+        } as TestCase,
+      },
+      {
+        testCase: {
+          vars: {
+            commonVar: 'override2',
+            test2Var: 'value2',
+          },
+        } as TestCase,
+      },
+    ] as any;
+
+    const header = getHeaderForTable(mockEval);
+
+    // Count occurrences of commonVar
+    const commonVarCount = header.vars.filter((name) => name === 'commonVar').length;
+    expect(commonVarCount).toBe(1);
+
+    // All vars should be present
+    expect(header.vars).toContain('commonVar');
+    expect(header.vars).toContain('defaultOnly');
+    expect(header.vars).toContain('test1Var');
+    expect(header.vars).toContain('test2Var');
+  });
+});

test/util/exportToFile/index.test.ts

@@ -0,0 +1,382 @@
+import { describe, expect, it } from 'vitest';
+import { ResultFailureReason } from '../../../src/types/index';
+import { getHeaderForTable } from '../../../src/util/exportToFile/getHeaderForTable';
+import {
+  convertEvalResultToTableCell,
+  convertTestResultsToTableRow,
+} from '../../../src/util/exportToFile/index';
+
+import type Eval from '../../../src/models/eval';
+import type EvalResult from '../../../src/models/evalResult';
+
+describe('exportToFile utils', () => {
+  describe('getHeaderForTable', () => {
+    it('should extract vars from defaultTest', () => {
+      const eval_: Partial<Eval> = {
+        id: 'test-id',
+        createdAt: Date.now(),
+        config: {
+          defaultTest: {
+            vars: {
+              var1: 'value1',
+              var2: 'value2',
+            },
+          },
+        },
+        prompts: [],
+        results: [],
+        persisted: false,
+      };
+
+      const result = getHeaderForTable(eval_ as Eval);
+      expect(result.vars).toEqual(['var1', 'var2']);
+    });
+
+    it('should extract vars from tests array', () => {
+      const eval_: Partial<Eval> = {
+        id: 'test-id',
+        createdAt: Date.now(),
+        config: {
+          tests: [
+            {
+              vars: {
+                var3: 'value3',
+                var4: 'value4',
+              },
+            },
+          ],
+        },
+        prompts: [],
+        results: [],
+        persisted: false,
+      };
+
+      const result = getHeaderForTable(eval_ as Eval);
+      expect(result.vars).toEqual(['var3', 'var4']);
+    });
+
+    it('should extract vars from scenarios', () => {
+      const eval_: Partial<Eval> = {
+        id: 'test-id',
+        createdAt: Date.now(),
+        config: {
+          scenarios: [
+            {
+              config: [
+                {
+                  vars: {
+                    var5: 'value5',
+                  },
+                },
+              ],
+              tests: [
+                {
+                  vars: {
+                    var6: 'value6',
+                  },
+                },
+              ],
+            },
+          ],
+        },
+        prompts: [],
+        results: [],
+        persisted: false,
+      };
+
+      const result = getHeaderForTable(eval_ as Eval);
+      expect(result.vars).toEqual(['var5', 'var6']);
+    });
+
+    it('should handle empty config', () => {
+      const eval_: Partial<Eval> = {
+        id: 'test-id',
+        createdAt: Date.now(),
+        config: {},
+        prompts: [],
+        results: [],
+        persisted: false,
+      };
+
+      const result = getHeaderForTable(eval_ as Eval);
+      expect(result.vars).toEqual([]);
+    });
+  });
+
+  describe('convertEvalResultToTableCell', () => {
+    it('should handle successful assertion result', () => {
+      const result: Partial<EvalResult> = {
+        id: 'test-1',
+        evalId: 'eval-1',
+        testCase: {
+          assert: [
+            {
+              type: 'contains',
+              value: 'test',
+            },
+          ],
+        },
+        success: true,
+        response: {
+          output: 'test output',
+        },
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        failureReason: ResultFailureReason.NONE,
+      };
+
+      const output = convertEvalResultToTableCell(result as EvalResult);
+      expect(output.text).toBe('test output');
+      expect(output.pass).toBe(true);
+    });
+
+    it('should handle failed assertion result', () => {
+      const result: Partial<EvalResult> = {
+        id: 'test-1',
+        evalId: 'eval-1',
+        testCase: {
+          assert: [
+            {
+              type: 'contains',
+              value: 'test',
+            },
+          ],
+        },
+        success: false,
+        error: 'test error',
+        response: {
+          output: 'test output',
+        },
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        gradingResult: {
+          pass: false,
+          score: 0,
+          reason: 'test failure',
+          componentResults: [
+            {
+              pass: false,
+              score: 0,
+              reason: 'test failure',
+            },
+          ],
+        },
+        failureReason: ResultFailureReason.ASSERT,
+      };
+
+      const output = convertEvalResultToTableCell(result as EvalResult);
+      expect(output.text).toBe('test output');
+      expect(output.pass).toBe(false);
+    });
+
+    it('should handle error without assertion', () => {
+      const result: Partial<EvalResult> = {
+        id: 'test-1',
+        evalId: 'eval-1',
+        testCase: {},
+        error: 'test error',
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        failureReason: ResultFailureReason.ERROR,
+      };
+
+      const output = convertEvalResultToTableCell(result as EvalResult);
+      expect(output.text).toBe('test error');
+    });
+
+    it('should preserve falsy outputs like 0 and false', () => {
+      const numericResult: Partial<EvalResult> = {
+        id: 'test-1',
+        evalId: 'eval-1',
+        testCase: {},
+        response: {
+          output: 0,
+        },
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        failureReason: ResultFailureReason.NONE,
+      };
+
+      const booleanResult: Partial<EvalResult> = {
+        id: 'test-2',
+        evalId: 'eval-1',
+        testCase: {},
+        response: {
+          output: false,
+        },
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        failureReason: ResultFailureReason.NONE,
+      };
+
+      const numericOutput = convertEvalResultToTableCell(numericResult as EvalResult);
+      const booleanOutput = convertEvalResultToTableCell(booleanResult as EvalResult);
+
+      expect(numericOutput.text).toBe('0');
+      expect(booleanOutput.text).toBe('false');
+    });
+
+    it('should handle null output by falling back to error', () => {
+      const resultWithError: Partial<EvalResult> = {
+        id: 'test-1',
+        evalId: 'eval-1',
+        testCase: {},
+        response: {
+          output: null,
+        },
+        error: 'Provider returned null',
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        failureReason: ResultFailureReason.ERROR,
+      };
+
+      const resultWithoutError: Partial<EvalResult> = {
+        id: 'test-2',
+        evalId: 'eval-1',
+        testCase: {},
+        response: {
+          output: null,
+        },
+        prompt: {
+          raw: 'test prompt',
+          label: 'test',
+        },
+        provider: {
+          id: 'test-provider',
+        },
+        failureReason: ResultFailureReason.NONE,
+      };
+
+      const outputWithError = convertEvalResultToTableCell(resultWithError as EvalResult);
+      const outputWithoutError = convertEvalResultToTableCell(resultWithoutError as EvalResult);
+
+      expect(outputWithError.text).toBe('Provider returned null');
+      expect(outputWithoutError.text).toBe('');
+    });
+  });
+
+  describe('convertTestResultsToTableRow', () => {
+    it('should convert results to table row', () => {
+      const results: Partial<EvalResult>[] = [
+        {
+          id: 'test-1',
+          evalId: 'eval-1',
+          description: 'test description',
+          promptIdx: 0,
+          testCase: {
+            vars: {
+              var1: 'value1',
+              var2: 'value2',
+            },
+          },
+          response: {
+            output: 'test output',
+          },
+          prompt: {
+            raw: 'test prompt',
+            label: 'test',
+          },
+          provider: {
+            id: 'test-provider',
+          },
+          failureReason: ResultFailureReason.NONE,
+        },
+      ];
+
+      const varsForHeader = ['var1', 'var2'];
+
+      const row = convertTestResultsToTableRow(results as EvalResult[], varsForHeader);
+
+      expect(row.description).toBe('test description');
+      expect(row.vars).toEqual(['value1', 'value2']);
+      expect(row.outputs[0].text).toBe('test output');
+    });
+
+    it('should handle complex var values', () => {
+      const results: Partial<EvalResult>[] = [
+        {
+          id: 'test-1',
+          evalId: 'eval-1',
+          promptIdx: 0,
+          testCase: {
+            vars: {
+              var1: { complex: 'object' },
+            },
+          },
+          prompt: {
+            raw: 'test prompt',
+            label: 'test',
+          },
+          provider: {
+            id: 'test-provider',
+          },
+          failureReason: ResultFailureReason.NONE,
+        },
+      ];
+
+      const varsForHeader = ['var1'];
+
+      const row = convertTestResultsToTableRow(results as EvalResult[], varsForHeader);
+      expect(row.vars).toEqual(['{"complex":"object"}']);
+    });
+
+    it('should preserve falsy var values like 0 and false', () => {
+      const results: Partial<EvalResult>[] = [
+        {
+          id: 'test-1',
+          evalId: 'eval-1',
+          promptIdx: 0,
+          testCase: {
+            vars: {
+              var1: 0,
+              var2: false,
+            },
+          },
+          prompt: {
+            raw: 'test prompt',
+            label: 'test',
+          },
+          provider: {
+            id: 'test-provider',
+          },
+          failureReason: ResultFailureReason.NONE,
+        },
+      ];
+
+      const varsForHeader = ['var1', 'var2'];
+
+      const row = convertTestResultsToTableRow(results as EvalResult[], varsForHeader);
+      expect(row.vars).toEqual(['0', 'false']);
+    });
+  });
+});

test/util/fetch/errors.test.ts

@@ -0,0 +1,533 @@
+import { describe, expect, it } from 'vitest';
+import {
+  extractRateLimitErrorCode,
+  findTargetErrorStatus,
+  formatRateLimitDetail,
+  formatRateLimitErrorMessage,
+  HttpRateLimitError,
+  isAbortError,
+  isHardQuotaCode,
+  isHttpRateLimitError,
+  isNonTransientHttpStatus,
+  isTransientConnectionError,
+} from '../../../src/util/fetch/errors';
+
+describe('isAbortError', () => {
+  it('returns true for AbortError and AbortException', () => {
+    const abortError = new Error('aborted');
+    abortError.name = 'AbortError';
+    const abortException = new Error('aborted');
+    abortException.name = 'AbortException';
+    expect(isAbortError(abortError)).toBe(true);
+    expect(isAbortError(abortException)).toBe(true);
+  });
+
+  it('returns false for other errors and non-errors', () => {
+    expect(isAbortError(new TypeError('terminated'))).toBe(false);
+    expect(isAbortError(new Error('boom'))).toBe(false);
+    expect(isAbortError({ name: 'AbortError' })).toBe(false);
+    expect(isAbortError('AbortError')).toBe(false);
+    expect(isAbortError(undefined)).toBe(false);
+  });
+});
+
+describe('isNonTransientHttpStatus', () => {
+  it('returns true for 401 Unauthorized', () => {
+    expect(isNonTransientHttpStatus(401)).toBe(true);
+  });
+
+  it('returns true for 403 Forbidden', () => {
+    expect(isNonTransientHttpStatus(403)).toBe(true);
+  });
+
+  it('returns true for 404 Not Found', () => {
+    expect(isNonTransientHttpStatus(404)).toBe(true);
+  });
+
+  it('returns false for 500 Internal Server Error (transient)', () => {
+    expect(isNonTransientHttpStatus(500)).toBe(false);
+  });
+
+  it('returns true for 501 Not Implemented', () => {
+    expect(isNonTransientHttpStatus(501)).toBe(true);
+  });
+
+  it('returns false for 200 OK', () => {
+    expect(isNonTransientHttpStatus(200)).toBe(false);
+  });
+
+  it('returns false for 201 Created', () => {
+    expect(isNonTransientHttpStatus(201)).toBe(false);
+  });
+
+  it('returns false for 429 Too Many Requests (transient)', () => {
+    expect(isNonTransientHttpStatus(429)).toBe(false);
+  });
+
+  it('returns false for 502 Bad Gateway (transient)', () => {
+    expect(isNonTransientHttpStatus(502)).toBe(false);
+  });
+
+  it('returns false for 503 Service Unavailable (transient)', () => {
+    expect(isNonTransientHttpStatus(503)).toBe(false);
+  });
+
+  it('returns false for 504 Gateway Timeout (transient)', () => {
+    expect(isNonTransientHttpStatus(504)).toBe(false);
+  });
+});
+
+describe('findTargetErrorStatus', () => {
+  it('returns undefined for empty results', () => {
+    expect(findTargetErrorStatus([])).toBeUndefined();
+  });
+
+  it('returns undefined when no HTTP status in results', () => {
+    const results = [{ response: {} }, { response: { metadata: {} } }];
+    expect(findTargetErrorStatus(results)).toBeUndefined();
+  });
+
+  it('returns undefined for successful HTTP status', () => {
+    const results = [{ response: { metadata: { http: { status: 200 } } } }];
+    expect(findTargetErrorStatus(results)).toBeUndefined();
+  });
+
+  it('returns undefined for transient errors (429, 502, 503, 504)', () => {
+    const results = [
+      { response: { metadata: { http: { status: 429 } } } },
+      { response: { metadata: { http: { status: 502 } } } },
+      { response: { metadata: { http: { status: 503 } } } },
+      { response: { metadata: { http: { status: 504 } } } },
+    ];
+    expect(findTargetErrorStatus(results)).toBeUndefined();
+  });
+
+  it('returns 401 for unauthorized error', () => {
+    const results = [
+      { response: { metadata: { http: { status: 200 } } } },
+      { response: { metadata: { http: { status: 401 } } } },
+    ];
+    expect(findTargetErrorStatus(results)).toBe(401);
+  });
+
+  it('returns 403 for forbidden error', () => {
+    const results = [{ response: { metadata: { http: { status: 403 } } } }];
+    expect(findTargetErrorStatus(results)).toBe(403);
+  });
+
+  it('returns 404 for not found error', () => {
+    const results = [{ response: { metadata: { http: { status: 404 } } } }];
+    expect(findTargetErrorStatus(results)).toBe(404);
+  });
+
+  it('returns undefined for 500 Internal Server Error (transient)', () => {
+    const results = [{ response: { metadata: { http: { status: 500 } } } }];
+    expect(findTargetErrorStatus(results)).toBeUndefined();
+  });
+
+  it('returns 501 for not implemented error', () => {
+    const results = [{ response: { metadata: { http: { status: 501 } } } }];
+    expect(findTargetErrorStatus(results)).toBe(501);
+  });
+
+  it('returns first non-transient error found', () => {
+    const results = [
+      { response: { metadata: { http: { status: 200 } } } },
+      { response: { metadata: { http: { status: 403 } } } },
+      { response: { metadata: { http: { status: 404 } } } },
+    ];
+    expect(findTargetErrorStatus(results)).toBe(403);
+  });
+});
+
+describe('isTransientConnectionError', () => {
+  it('returns false for undefined error', () => {
+    expect(isTransientConnectionError(undefined)).toBe(false);
+  });
+
+  it('returns true for ECONNRESET errors', () => {
+    const error = new Error('Connection reset') as Error & { code?: string };
+    error.code = 'ECONNRESET';
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for mixed-case ECONNRESET messages', () => {
+    const error = new Error('EConnReset');
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for EPIPE errors', () => {
+    const error = new Error('Broken pipe') as Error & { code?: string };
+    error.code = 'EPIPE';
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for socket hang up errors', () => {
+    const error = new Error('socket hang up');
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for mixed-case socket hang up errors', () => {
+    const error = new Error('Socket Hang Up');
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for bad record mac errors', () => {
+    const error = new Error('bad record mac');
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for mixed-case bad record mac errors', () => {
+    const error = new Error('Bad Record MAC');
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns true for standalone eproto errors', () => {
+    const error = new Error('eproto');
+    expect(isTransientConnectionError(error)).toBe(true);
+  });
+
+  it('returns false for permanent TLS config errors', () => {
+    const error = new Error('eproto self signed certificate');
+    expect(isTransientConnectionError(error)).toBe(false);
+  });
+
+  it('returns false for eproto unable-to-verify TLS errors', () => {
+    // Must include `eproto` to enter the permanent-error exclusion; otherwise the
+    // message has no transient marker and would return false trivially.
+    const error = new Error('write EPROTO unable to verify the first certificate');
+    expect(isTransientConnectionError(error)).toBe(false);
+  });
+
+  it('returns false for eproto unknown ca TLS errors', () => {
+    const error = new Error('write EPROTO tlsv1 alert unknown ca');
+    expect(isTransientConnectionError(error)).toBe(false);
+  });
+
+  it('returns false for eproto certificate verify TLS errors', () => {
+    const error = new Error('write EPROTO certificate verify failed');
+    expect(isTransientConnectionError(error)).toBe(false);
+  });
+
+  it('returns false for wrong version number errors', () => {
+    const error = new Error('eproto wrong version number');
+    expect(isTransientConnectionError(error)).toBe(false);
+  });
+});
+
+describe('isHardQuotaCode', () => {
+  it.each([
+    ['insufficient_quota', true],
+    ['billing_hard_limit_reached', true],
+    ['billing_not_active', true],
+    ['access_terminated', true],
+    ['quota_exceeded', true],
+    ['rate_limit_exceeded', false],
+    ['tokens_per_min', false],
+    ['', false],
+  ])('isHardQuotaCode(%s) === %s', (code, expected) => {
+    expect(isHardQuotaCode(code as string)).toBe(expected);
+  });
+
+  it('returns false for undefined', () => {
+    expect(isHardQuotaCode(undefined)).toBe(false);
+  });
+});
+
+describe('extractRateLimitErrorCode', () => {
+  it('extracts OpenAI / Azure shape: { error: { code } }', () => {
+    expect(
+      extractRateLimitErrorCode({
+        error: { code: 'insufficient_quota', message: 'You exceeded your current quota' },
+      }),
+    ).toBe('insufficient_quota');
+  });
+
+  it('falls back to error.type when error.code is missing', () => {
+    expect(extractRateLimitErrorCode({ error: { type: 'rate_limit_error' } })).toBe(
+      'rate_limit_error',
+    );
+  });
+
+  it('reads top-level code', () => {
+    expect(extractRateLimitErrorCode({ code: 'tokens_per_min' })).toBe('tokens_per_min');
+  });
+
+  it('reads top-level type', () => {
+    expect(extractRateLimitErrorCode({ type: 'rate_limit_error' })).toBe('rate_limit_error');
+  });
+
+  it('returns undefined for non-object', () => {
+    expect(extractRateLimitErrorCode('plain text')).toBeUndefined();
+    expect(extractRateLimitErrorCode(null)).toBeUndefined();
+    expect(extractRateLimitErrorCode(undefined)).toBeUndefined();
+    expect(extractRateLimitErrorCode(123)).toBeUndefined();
+  });
+
+  it('returns undefined when no code-like field is present', () => {
+    expect(extractRateLimitErrorCode({ error: {} })).toBeUndefined();
+    expect(extractRateLimitErrorCode({})).toBeUndefined();
+  });
+
+  it('ignores empty string code', () => {
+    expect(extractRateLimitErrorCode({ error: { code: '' } })).toBeUndefined();
+  });
+});
+
+describe('HttpRateLimitError', () => {
+  it('classifies known quota codes as kind="quota"', () => {
+    const err = new HttpRateLimitError({ status: 429, code: 'insufficient_quota' });
+    expect(err.kind).toBe('quota');
+    expect(err.message).toContain('Quota exceeded');
+    expect(err.message).toContain('429');
+    expect(err.message).toContain('insufficient_quota');
+  });
+
+  it('classifies unknown / per-window codes as kind="rate_limit"', () => {
+    const err = new HttpRateLimitError({ status: 429, code: 'rate_limit_exceeded' });
+    expect(err.kind).toBe('rate_limit');
+    expect(err.message).toContain('Rate limit exceeded');
+    expect(err.message).toContain('429');
+  });
+
+  it('preserves status, retryAfterMs, resetAt, headers, body', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      statusText: 'Too Many Requests',
+      retryAfterMs: 5000,
+      resetAt: 1_700_000_000_000,
+      headers: { 'retry-after': '5' },
+      body: { error: { code: 'rate_limit_exceeded' } },
+      code: 'rate_limit_exceeded',
+    });
+    expect(err.status).toBe(429);
+    expect(err.statusText).toBe('Too Many Requests');
+    expect(err.retryAfterMs).toBe(5000);
+    expect(err.resetAt).toBe(1_700_000_000_000);
+    expect(err.headers?.['retry-after']).toBe('5');
+    expect(err.body).toEqual({ error: { code: 'rate_limit_exceeded' } });
+    expect(err.code).toBe('rate_limit_exceeded');
+  });
+
+  it('defaults statusText to "Too Many Requests"', () => {
+    const err = new HttpRateLimitError({ status: 429 });
+    expect(err.statusText).toBe('Too Many Requests');
+  });
+
+  it('defaults empty statusText to "Too Many Requests"', () => {
+    const err = new HttpRateLimitError({ status: 429, statusText: '' });
+    expect(err.statusText).toBe('Too Many Requests');
+    expect(formatRateLimitErrorMessage(err)).toContain('Too Many Requests');
+  });
+
+  it('produces a message containing the substrings legacy classifiers match on', () => {
+    const err = new HttpRateLimitError({ status: 429, code: 'rate_limit_exceeded' });
+    const lowered = err.message.toLowerCase();
+    // Back-compat: substring matchers across the codebase look for these tokens
+    expect(err.message).toContain('429');
+    expect(lowered.includes('rate limit') || lowered.includes('too many requests')).toBe(true);
+  });
+
+  it('isHttpRateLimitError type guard works', () => {
+    expect(isHttpRateLimitError(new HttpRateLimitError({ status: 429 }))).toBe(true);
+    expect(isHttpRateLimitError(new Error('rate limit'))).toBe(false);
+    expect(isHttpRateLimitError(undefined)).toBe(false);
+    expect(isHttpRateLimitError(null)).toBe(false);
+    expect(isHttpRateLimitError({ name: 'HttpRateLimitError' })).toBe(false);
+  });
+
+  it('shallow-copies headers so post-construction mutation does not leak in', () => {
+    const headers = { 'retry-after': '5' };
+    const err = new HttpRateLimitError({ status: 429, headers });
+    headers['retry-after'] = '999';
+    expect(err.headers?.['retry-after']).toBe('5');
+  });
+
+  it('rejects negative retryAfterMs', () => {
+    const err = new HttpRateLimitError({ status: 429, retryAfterMs: -100 });
+    expect(err.retryAfterMs).toBeUndefined();
+  });
+
+  it('keeps a valid resetAt even when retryAfterMs is negative (independent validation)', () => {
+    const resetAt = Date.now() + 30_000;
+    const err = new HttpRateLimitError({
+      status: 429,
+      retryAfterMs: -100,
+      resetAt,
+    });
+    expect(err.retryAfterMs).toBeUndefined();
+    expect(err.resetAt).toBe(resetAt);
+  });
+
+  it('drops a negative resetAt when retryAfterMs is valid', () => {
+    const err = new HttpRateLimitError({ status: 429, retryAfterMs: 5000, resetAt: -1 });
+    expect(err.retryAfterMs).toBe(5000);
+    expect(err.resetAt).toBeUndefined();
+  });
+
+  it('drops a non-number resetAt (e.g. string) regardless of retryAfterMs', () => {
+    // Exercises the runtime typeof guard for untyped / `any` callers.
+    const err = new HttpRateLimitError({
+      status: 429,
+      resetAt: '1700000000000' as unknown as number,
+    });
+    expect(err.resetAt).toBeUndefined();
+  });
+
+  it('drops non-finite retry metadata', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      retryAfterMs: Number.POSITIVE_INFINITY,
+      resetAt: Number.POSITIVE_INFINITY,
+    });
+
+    expect(err.retryAfterMs).toBeUndefined();
+    expect(err.resetAt).toBeUndefined();
+    expect(formatRateLimitDetail(err)).toBe('');
+  });
+});
+
+describe('formatRateLimitDetail', () => {
+  it('renders retry-after seconds', () => {
+    const err = new HttpRateLimitError({ status: 429, retryAfterMs: 12_000 });
+    expect(formatRateLimitDetail(err)).toBe(' [retry after 12s]');
+  });
+
+  it('renders resetAt fallback when retryAfterMs is missing', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      resetAt: Date.now() + 30_000,
+    });
+    expect(formatRateLimitDetail(err)).toMatch(/resets in \d+s/);
+  });
+
+  it('prefers retry-after over resetAt when both are present', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      retryAfterMs: 12_000,
+      resetAt: Date.now() + 999_000,
+    });
+    expect(formatRateLimitDetail(err)).toBe(' [retry after 12s]');
+  });
+
+  it('returns empty string when no metadata is present', () => {
+    const err = new HttpRateLimitError({ status: 429 });
+    expect(formatRateLimitDetail(err)).toBe('');
+  });
+
+  it('returns empty string for kind=quota even when retry metadata is present', () => {
+    // Quota errors should not advertise a "retry after Xs" hint — that
+    // would contradict the "Retries will not help" message the providers
+    // surface to the operator. Use a 2-hour retryAfterMs so the constructor's
+    // "small Retry-After downgrades quota to rate_limit" heuristic does not
+    // fire here.
+    const twoHoursMs = 2 * 60 * 60 * 1000;
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'insufficient_quota',
+      retryAfterMs: twoHoursMs,
+      resetAt: Date.now() + twoHoursMs,
+    });
+    expect(err.kind).toBe('quota');
+    expect(formatRateLimitDetail(err)).toBe('');
+  });
+});
+
+describe('HttpRateLimitError: small Retry-After downgrades quota to rate_limit', () => {
+  // Azure OpenAI returns `insufficient_quota` for both billing exhaustion and
+  // per-minute deployment saturation. A small Retry-After is the server
+  // hinting at recovery — billing quotas don't recover in seconds.
+  it('downgrades insufficient_quota with small retryAfterMs to rate_limit', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'insufficient_quota',
+      retryAfterMs: 30_000,
+    });
+    expect(err.kind).toBe('rate_limit');
+    expect(err.code).toBe('insufficient_quota');
+    expect(err.message).toContain('Rate limit exceeded');
+  });
+
+  it('keeps kind=quota when no Retry-After is present', () => {
+    const err = new HttpRateLimitError({ status: 429, code: 'insufficient_quota' });
+    expect(err.kind).toBe('quota');
+  });
+
+  it('keeps kind=quota when Retry-After is large (> 1h)', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'insufficient_quota',
+      retryAfterMs: 90 * 60 * 1000,
+    });
+    expect(err.kind).toBe('quota');
+  });
+
+  it('downgrades all hard-quota codes when Retry-After is small', () => {
+    for (const code of ['quota_exceeded', 'billing_hard_limit_reached', 'insufficient_quota']) {
+      const err = new HttpRateLimitError({ status: 429, code, retryAfterMs: 5000 });
+      expect(err.kind).toBe('rate_limit');
+    }
+  });
+});
+
+describe('formatRateLimitErrorMessage', () => {
+  it('formats a per-window rate limit with status, code, and retry-after', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'rate_limit_exceeded',
+      retryAfterMs: 7000,
+    });
+    expect(formatRateLimitErrorMessage(err)).toBe(
+      'Rate limit exceeded: HTTP 429 Too Many Requests (code: rate_limit_exceeded) [retry after 7s]',
+    );
+  });
+
+  it('formats a hard quota with the non-retryable hint', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'insufficient_quota',
+    });
+    expect(formatRateLimitErrorMessage(err)).toBe(
+      'Quota exceeded: HTTP 429 Too Many Requests (code: insufficient_quota). Retries will not help — check your billing or daily quota.',
+    );
+  });
+
+  it('omits the code segment when no code is set', () => {
+    const err = new HttpRateLimitError({ status: 429 });
+    expect(formatRateLimitErrorMessage(err)).toBe(
+      'Rate limit exceeded: HTTP 429 Too Many Requests',
+    );
+  });
+
+  it('appends `details` for upstream-supplied context', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'rate_limit_exceeded',
+      retryAfterMs: 12_000,
+    });
+    const out = formatRateLimitErrorMessage(
+      err,
+      'Rate limit reached for gpt-4o (current: 1000 TPM)',
+    );
+    expect(out).toBe(
+      'Rate limit exceeded: HTTP 429 Too Many Requests (code: rate_limit_exceeded) Rate limit reached for gpt-4o (current: 1000 TPM) [retry after 12s]',
+    );
+  });
+
+  it('appends `details` before the non-retryable hint on quota errors', () => {
+    const err = new HttpRateLimitError({
+      status: 429,
+      code: 'insufficient_quota',
+    });
+    expect(formatRateLimitErrorMessage(err, 'Quota exhausted for asst_xyz')).toBe(
+      'Quota exceeded: HTTP 429 Too Many Requests (code: insufficient_quota) Quota exhausted for asst_xyz. Retries will not help — check your billing or daily quota.',
+    );
+  });
+
+  it('produces a single non-redundant prefix (no double "Rate limit exceeded")', () => {
+    const err = new HttpRateLimitError({ status: 429, code: 'rate_limit_exceeded' });
+    const out = formatRateLimitErrorMessage(err);
+    expect(out.match(/Rate limit exceeded/g)?.length).toBe(1);
+    expect(out.match(/HTTP 429/g)?.length).toBe(1);
+  });
+});

test/util/fetch/stripDecompressionHeaders.test.ts

@@ -0,0 +1,240 @@
+import { describe, expect, it } from 'vitest';
+import { stripDecompressionHeaders } from '../../../src/util/fetch/stripDecompressionHeaders';
+import type { Dispatcher } from 'undici';
+
+type RawHeaderPairs = [string, string][];
+type ResponseCallbackMode = 'started' | 'start' | 'both';
+type OnResponseStartArgs = Parameters<NonNullable<Dispatcher.DispatchHandler['onResponseStart']>>;
+type ParsedHeaders = OnResponseStartArgs[2];
+type TrackedEvent =
+  | {
+      method: 'onResponseStart';
+      args: OnResponseStartArgs;
+    }
+  | {
+      method: 'onResponseStarted';
+      args: [];
+    };
+
+interface DispatchResponseStartOptions {
+  rawHeaders?: Buffer[] | null;
+  parsedHeaders: ParsedHeaders;
+  statusCode?: number;
+  dispatchResult?: boolean;
+  callbackMode?: ResponseCallbackMode;
+}
+
+interface DispatchResponseStartResult {
+  controller: Dispatcher.DispatchController & { rawHeaders?: Buffer[] | null };
+  events: TrackedEvent[];
+  dispatched: boolean;
+}
+
+function makeRawHeaders(pairs: RawHeaderPairs): Buffer[] {
+  return pairs.flatMap(([name, value]) => [Buffer.from(name), Buffer.from(value)]);
+}
+
+function rawHeadersToPairs(rawHeaders: Buffer[]): RawHeaderPairs {
+  const pairs: RawHeaderPairs = [];
+  for (let i = 0; i + 1 < rawHeaders.length; i += 2) {
+    pairs.push([rawHeaders[i].toString(), rawHeaders[i + 1].toString()]);
+  }
+  return pairs;
+}
+
+/**
+ * Drives the strip interceptor with a synthetic controller the way undici's
+ * v7→v8 bridge does on Node 26: the controller carries the original raw
+ * headers while the parsed headers reflect any rewriting the decompress
+ * interceptor performed. This exercises the strip logic on every Node
+ * version, without needing a Node 26 fetch.
+ */
+function dispatchResponseStart({
+  rawHeaders,
+  parsedHeaders,
+  statusCode = 200,
+  dispatchResult = true,
+  callbackMode = 'both',
+}: DispatchResponseStartOptions): DispatchResponseStartResult {
+  const events: TrackedEvent[] = [];
+  const innerHandler: Dispatcher.DispatchHandler = {
+    onResponseStart(controller, status, headers, statusMessage) {
+      events.push({
+        method: 'onResponseStart',
+        args: [controller, status, headers, statusMessage],
+      });
+    },
+    onResponseStarted() {
+      events.push({ method: 'onResponseStarted', args: [] });
+    },
+  };
+  const controller = { rawHeaders } as unknown as Dispatcher.DispatchController & {
+    rawHeaders?: Buffer[] | null;
+  };
+  // Real undici handlers use one callback generation. Tests can select either
+  // generation explicitly; "both" is only a compact harness mode.
+  const dispatch: Parameters<Dispatcher.DispatchInterceptor>[0] = (_opts, handler) => {
+    if (callbackMode === 'both' || callbackMode === 'started') {
+      handler.onResponseStarted?.();
+    }
+    if (callbackMode === 'both' || callbackMode === 'start') {
+      handler.onResponseStart?.(controller, statusCode, parsedHeaders, 'OK');
+    }
+    return dispatchResult;
+  };
+  const composed = stripDecompressionHeaders()(dispatch);
+  const dispatched = composed(
+    { path: '/', method: 'GET' } as Dispatcher.DispatchOptions,
+    innerHandler,
+  );
+  return { controller, events, dispatched };
+}
+
+describe('stripDecompressionHeaders', () => {
+  it('strips content-encoding and content-length from rawHeaders when decompress decoded the body', () => {
+    // decompress removed both headers from the parsed headers => body is decoded.
+    const { controller, events } = dispatchResponseStart({
+      rawHeaders: makeRawHeaders([
+        ['content-type', 'application/json'],
+        ['content-encoding', 'gzip'],
+        ['content-length', '123'],
+        ['x-request-id', 'abc'],
+      ]),
+      parsedHeaders: { 'content-type': 'application/json', 'x-request-id': 'abc' },
+    });
+
+    expect(rawHeadersToPairs(controller.rawHeaders as Buffer[])).toEqual([
+      ['content-type', 'application/json'],
+      ['x-request-id', 'abc'],
+    ]);
+    expect(events.map((e) => e.method)).toEqual(['onResponseStarted', 'onResponseStart']);
+  });
+
+  it('strips mixed-case and repeated content-encoding entries', () => {
+    // Defensive: undici's parseHeaders merges repeated names into one array
+    // value and decompress rejects array-valued content-encoding, so this
+    // exact state is synthetic — the strip should still handle it sanely.
+    const { controller } = dispatchResponseStart({
+      rawHeaders: makeRawHeaders([
+        ['Content-Encoding', 'gzip'],
+        ['content-encoding', 'br'],
+        ['Content-Length', '99'],
+        ['Content-Type', 'text/plain'],
+      ]),
+      parsedHeaders: { 'content-type': 'text/plain' },
+    });
+
+    expect(rawHeadersToPairs(controller.rawHeaders as Buffer[])).toEqual([
+      ['Content-Type', 'text/plain'],
+    ]);
+  });
+
+  it('leaves rawHeaders untouched when the parsed headers still carry content-encoding (decompress skipped)', () => {
+    // An unsupported encoding makes the decompress interceptor pass the
+    // response through with its original headers. The raw content-encoding
+    // must survive so callers can tell the body is still encoded.
+    const rawHeaders = makeRawHeaders([
+      ['content-encoding', 'x-snappy'],
+      ['content-length', '40'],
+    ]);
+    const { controller } = dispatchResponseStart({
+      rawHeaders,
+      parsedHeaders: { 'content-encoding': 'x-snappy', 'content-length': '40' },
+    });
+
+    expect(controller.rawHeaders).toBe(rawHeaders);
+    expect(rawHeadersToPairs(controller.rawHeaders as Buffer[])).toEqual([
+      ['content-encoding', 'x-snappy'],
+      ['content-length', '40'],
+    ]);
+  });
+
+  it('leaves content-length untouched on responses that were never compressed', () => {
+    const rawHeaders = makeRawHeaders([
+      ['content-type', 'application/json'],
+      ['content-length', '123'],
+    ]);
+    const { controller } = dispatchResponseStart({
+      rawHeaders,
+      parsedHeaders: { 'content-type': 'application/json', 'content-length': '123' },
+    });
+
+    expect(controller.rawHeaders).toBe(rawHeaders);
+    expect(rawHeadersToPairs(controller.rawHeaders as Buffer[])).toEqual([
+      ['content-type', 'application/json'],
+      ['content-length', '123'],
+    ]);
+  });
+
+  it.each([
+    undefined,
+    null,
+  ])('passes through when controller.rawHeaders is %s (not an array)', (rawHeaders) => {
+    const { controller, events } = dispatchResponseStart({
+      rawHeaders,
+      parsedHeaders: { 'content-type': 'application/json' },
+    });
+
+    expect(controller.rawHeaders).toBe(rawHeaders);
+    expect(events.map((e) => e.method)).toEqual(['onResponseStarted', 'onResponseStart']);
+  });
+
+  it('blocks the strip when the parsed content-encoding key is not lowercase', () => {
+    // Decompress only decodes on the exact lowercase key, so a differently
+    // cased parsed key means the body was not decoded — the case-insensitive
+    // fallback in the gate must block the strip.
+    const rawHeaders = makeRawHeaders([
+      ['content-encoding', 'x-snappy'],
+      ['content-length', '40'],
+    ]);
+    const { controller } = dispatchResponseStart({
+      rawHeaders,
+      parsedHeaders: { 'Content-Encoding': 'x-snappy' },
+    });
+
+    expect(controller.rawHeaders).toBe(rawHeaders);
+  });
+
+  it.each([true, false])('propagates the dispatch backpressure result (%s)', (dispatchResult) => {
+    const { dispatched } = dispatchResponseStart({
+      rawHeaders: makeRawHeaders([['content-type', 'text/plain']]),
+      parsedHeaders: { 'content-type': 'text/plain' },
+      dispatchResult,
+    });
+
+    expect(dispatched).toBe(dispatchResult);
+  });
+
+  it('preserves a trailing unpaired rawHeaders element when stripping', () => {
+    const rawHeaders = [
+      ...makeRawHeaders([
+        ['content-encoding', 'gzip'],
+        ['x-request-id', 'abc'],
+      ]),
+      Buffer.from('dangling'),
+    ];
+    const { controller } = dispatchResponseStart({
+      rawHeaders,
+      parsedHeaders: { 'x-request-id': 'abc' },
+    });
+
+    const result = controller.rawHeaders as Buffer[];
+    expect(rawHeadersToPairs(result)).toEqual([['x-request-id', 'abc']]);
+    expect(result[result.length - 1].toString()).toBe('dangling');
+  });
+
+  it('forwards response events and arguments to the inner handler unchanged', () => {
+    const parsedHeaders = { 'content-type': 'application/json' };
+    const { controller, events } = dispatchResponseStart({
+      rawHeaders: makeRawHeaders([['content-type', 'application/json']]),
+      parsedHeaders,
+      callbackMode: 'start',
+    });
+
+    const start = events.find((e) => e.method === 'onResponseStart');
+    expect(start?.args[0]).toBe(controller);
+    expect(start?.args[1]).toBe(200);
+    expect(start?.args[2]).toBe(parsedHeaders);
+    expect(start?.args[3]).toBe('OK');
+  });
+});

test/util/fileReference.test.ts

@@ -0,0 +1,335 @@
+import fs from 'fs';
+import path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import * as esmModule from '../../src/esm';
+import logger from '../../src/logger';
+import * as pythonUtils from '../../src/python/pythonUtils';
+import { isJavascriptFile } from '../../src/util/fileExtensions';
+import { loadFileReference, processConfigFileReferences } from '../../src/util/fileReference';
+import { loadYaml } from '../../src/util/yamlLoad';
+import type { Logger } from 'winston';
+
+vi.mock('fs', async () => {
+  const actual = await vi.importActual<typeof import('fs')>('fs');
+  return {
+    ...actual,
+    default: {
+      ...actual,
+      promises: {
+        readFile: vi.fn(),
+      },
+    },
+    promises: {
+      readFile: vi.fn(),
+    },
+  };
+});
+vi.mock('path');
+vi.mock('../../src/util/yamlLoad');
+vi.mock('../../src/esm', () => ({
+  importModule: vi.fn(),
+}));
+vi.mock('../../src/python/pythonUtils', () => ({
+  runPython: vi.fn(),
+}));
+vi.mock('../../src/util/fileExtensions');
+vi.mock('../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+const importModule = vi.mocked(esmModule.importModule);
+const runPython = vi.mocked(pythonUtils.runPython);
+const readFileMock = vi.mocked(fs.promises.readFile);
+
+describe('fileReference utility functions', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+
+    vi.mocked(logger.debug).mockImplementation((_message: string) => {
+      return {
+        debug: vi.fn(),
+        info: vi.fn(),
+        warn: vi.fn(),
+        error: vi.fn(),
+      } as unknown as Logger;
+    });
+
+    vi.mocked(logger.error).mockImplementation((_message: string) => {
+      return {
+        debug: vi.fn(),
+        info: vi.fn(),
+        warn: vi.fn(),
+        error: vi.fn(),
+      } as unknown as Logger;
+    });
+
+    vi.mocked(path.resolve).mockImplementation((basePath, filePath) =>
+      filePath?.startsWith('/') ? filePath : path.join(basePath || '', filePath || ''),
+    );
+
+    vi.mocked(path.join).mockImplementation((...parts) => parts.filter(Boolean).join('/'));
+
+    vi.mocked(path.extname).mockImplementation((filePath) => {
+      if (!filePath) {
+        return '';
+      }
+      const parts = filePath.split('.');
+      return parts.length > 1 ? `.${parts[parts.length - 1]}` : '';
+    });
+
+    vi.mocked(isJavascriptFile).mockImplementation((filePath) => {
+      if (!filePath) {
+        return false;
+      }
+      return ['.js', '.mjs', '.ts', '.cjs'].some((ext) => filePath.endsWith(ext));
+    });
+
+    importModule.mockResolvedValue({});
+    runPython.mockResolvedValue({});
+  });
+
+  describe('loadFileReference', () => {
+    it('should load JSON files correctly', async () => {
+      const fileRef = 'file:///path/to/config.json';
+      const fileContent = '{"name": "test", "value": 42}';
+      const parsedContent = { name: 'test', value: 42 };
+
+      readFileMock.mockResolvedValue(fileContent);
+      vi.spyOn(JSON, 'parse').mockReturnValue(parsedContent);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(fs.promises.readFile).toHaveBeenCalledWith('/path/to/config.json', 'utf8');
+      expect(result).toEqual(parsedContent);
+    });
+
+    it('should load YAML files correctly', async () => {
+      const fileRef = 'file:///path/to/config.yaml';
+      const fileContent = 'name: test\nvalue: 42';
+      const parsedContent = { name: 'test', value: 42 };
+
+      readFileMock.mockResolvedValue(fileContent);
+      vi.mocked(loadYaml).mockReturnValue(parsedContent);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(fs.promises.readFile).toHaveBeenCalledWith('/path/to/config.yaml', 'utf8');
+      expect(loadYaml).toHaveBeenCalledWith(fileContent);
+      expect(result).toEqual(parsedContent);
+    });
+
+    it('should load JavaScript files correctly', async () => {
+      const fileRef = 'file:///path/to/config.js';
+      const moduleOutput = { settings: { temperature: 0.7 } };
+
+      vi.mocked(isJavascriptFile).mockReturnValue(true);
+      importModule.mockResolvedValue(moduleOutput);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(importModule).toHaveBeenCalledWith('/path/to/config.js', undefined);
+      expect(result).toEqual(moduleOutput);
+    });
+
+    it('should load JavaScript files with function name correctly', async () => {
+      const fileRef = 'file:///path/to/config.js:getConfig';
+      const moduleOutput = { getConfig: 'success' };
+
+      vi.mocked(isJavascriptFile).mockReturnValue(true);
+      importModule.mockResolvedValue(moduleOutput);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(importModule).toHaveBeenCalledWith('/path/to/config.js', 'getConfig');
+      expect(result).toEqual(moduleOutput);
+    });
+
+    it('should load Python files correctly', async () => {
+      const fileRef = 'file:///path/to/config.py';
+      const pythonOutput = { message: 'Hello from Python' };
+
+      runPython.mockResolvedValue(pythonOutput);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(runPython).toHaveBeenCalledWith('/path/to/config.py', 'get_config', []);
+      expect(result).toEqual(pythonOutput);
+    });
+
+    it('should load Python files with function name correctly', async () => {
+      const fileRef = 'file:///path/to/config.py:custom_func';
+      const pythonOutput = { custom: true };
+
+      runPython.mockResolvedValue(pythonOutput);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(runPython).toHaveBeenCalledWith('/path/to/config.py', 'custom_func', []);
+      expect(result).toEqual(pythonOutput);
+    });
+
+    it('should load text files correctly', async () => {
+      const fileRef = 'file:///path/to/config.txt';
+      const fileContent = 'This is a text file';
+
+      readFileMock.mockResolvedValue(fileContent);
+
+      const result = await loadFileReference(fileRef);
+
+      expect(fs.promises.readFile).toHaveBeenCalledWith('/path/to/config.txt', 'utf8');
+      expect(result).toEqual(fileContent);
+    });
+
+    it('should resolve file paths relative to the basePath', async () => {
+      const fileRef = 'file://config.json';
+      const basePath = '/base/path';
+      const fileContent = '{"name": "test"}';
+      const parsedContent = { name: 'test' };
+
+      vi.mocked(path.resolve).mockReturnValue('/base/path/config.json');
+      readFileMock.mockResolvedValue(fileContent);
+      vi.spyOn(JSON, 'parse').mockReturnValue(parsedContent);
+
+      const result = await loadFileReference(fileRef, basePath);
+
+      expect(path.resolve).toHaveBeenCalledWith('/base/path', 'config.json');
+      expect(fs.promises.readFile).toHaveBeenCalledWith('/base/path/config.json', 'utf8');
+      expect(result).toEqual(parsedContent);
+    });
+
+    it('should throw an error for unsupported file types', async () => {
+      const fileRef = 'file:///path/to/file.xyz';
+
+      vi.mocked(path.extname).mockReturnValue('.xyz');
+      vi.mocked(isJavascriptFile).mockReturnValue(false);
+
+      await expect(loadFileReference(fileRef)).rejects.toThrow('Unsupported file extension: .xyz');
+    });
+  });
+
+  describe('processConfigFileReferences', () => {
+    it('should return primitive values as is', async () => {
+      await expect(processConfigFileReferences(42)).resolves.toBe(42);
+      await expect(processConfigFileReferences('test')).resolves.toBe('test');
+      await expect(processConfigFileReferences(true)).resolves.toBe(true);
+      await expect(processConfigFileReferences(null)).resolves.toBeNull();
+      await expect(processConfigFileReferences(undefined)).resolves.toBeUndefined();
+    });
+
+    it('should process a simple file reference string', async () => {
+      const config = 'file:///path/to/config.json';
+      const parsedContent = { name: 'test', value: 42 };
+
+      readFileMock.mockResolvedValue(JSON.stringify(parsedContent));
+      vi.spyOn(JSON, 'parse').mockReturnValue(parsedContent);
+
+      const result = await processConfigFileReferences(config);
+
+      expect(result).toEqual(parsedContent);
+    });
+
+    it('should process nested file references in objects', async () => {
+      const config = {
+        setting1: 'value1',
+        setting2: 'file:///path/to/setting2.json',
+        nested: {
+          setting3: 'file:///path/to/setting3.yaml',
+        },
+      };
+
+      readFileMock.mockImplementation((path) => {
+        if (path === '/path/to/setting2.json') {
+          return Promise.resolve('{"key": "value2"}');
+        }
+        if (path === '/path/to/setting3.yaml') {
+          return Promise.resolve('key: value3');
+        }
+        return Promise.resolve('');
+      });
+
+      vi.spyOn(JSON, 'parse').mockImplementation((content) => {
+        if (content === '{"key": "value2"}') {
+          return { key: 'value2' };
+        }
+        return {};
+      });
+
+      vi.mocked(loadYaml).mockImplementation((content) => {
+        if (content === 'key: value3') {
+          return { key: 'value3' };
+        }
+        return {};
+      });
+
+      const result = await processConfigFileReferences(config);
+
+      expect(result).toEqual({
+        setting1: 'value1',
+        setting2: { key: 'value2' },
+        nested: {
+          setting3: { key: 'value3' },
+        },
+      });
+    });
+
+    it('should process file references in arrays', async () => {
+      const config = [
+        'regular string',
+        'file:///path/to/item1.json',
+        'file:///path/to/item2.yaml',
+        42,
+      ];
+
+      readFileMock.mockImplementation((path) => {
+        if (path === '/path/to/item1.json') {
+          return Promise.resolve('{"name": "item1"}');
+        }
+        if (path === '/path/to/item2.yaml') {
+          return Promise.resolve('name: item2');
+        }
+        return Promise.resolve('');
+      });
+
+      vi.spyOn(JSON, 'parse').mockImplementation((content) => {
+        if (content === '{"name": "item1"}') {
+          return { name: 'item1' };
+        }
+        return {};
+      });
+
+      vi.mocked(loadYaml).mockImplementation((content) => {
+        if (content === 'name: item2') {
+          return { name: 'item2' };
+        }
+        return {};
+      });
+
+      const result = await processConfigFileReferences(config);
+
+      expect(result).toEqual(['regular string', { name: 'item1' }, { name: 'item2' }, 42]);
+    });
+
+    it('should handle errors when processing file references', async () => {
+      const config = {
+        valid: 'regular value',
+        invalid: 'file:///path/to/nonexistent.json',
+      };
+
+      readFileMock.mockImplementation((path) => {
+        if (path === '/path/to/nonexistent.json') {
+          return Promise.reject(new Error('File not found'));
+        }
+        return Promise.resolve('');
+      });
+
+      await expect(processConfigFileReferences(config)).rejects.toThrow('File not found');
+      expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('File not found'));
+    });
+  });
+});

test/util/finishReason.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeFinishReason } from '../../src/util/finishReason';
+
+describe('normalizeFinishReason', () => {
+  describe('OpenAI mappings', () => {
+    it('should pass through OpenAI standard reasons unchanged', () => {
+      expect(normalizeFinishReason('stop')).toBe('stop');
+      expect(normalizeFinishReason('length')).toBe('length');
+      expect(normalizeFinishReason('content_filter')).toBe('content_filter');
+      expect(normalizeFinishReason('tool_calls')).toBe('tool_calls');
+    });
+
+    it('should map function_call to tool_calls', () => {
+      expect(normalizeFinishReason('function_call')).toBe('tool_calls');
+    });
+  });
+
+  describe('Anthropic mappings', () => {
+    it('should normalize Anthropic reasons to standard values', () => {
+      expect(normalizeFinishReason('end_turn')).toBe('stop');
+      expect(normalizeFinishReason('stop_sequence')).toBe('stop');
+      expect(normalizeFinishReason('max_tokens')).toBe('length');
+      expect(normalizeFinishReason('tool_use')).toBe('tool_calls');
+      expect(normalizeFinishReason('refusal')).toBe('content_filter');
+    });
+
+    it('should preserve pause_turn as-is', () => {
+      expect(normalizeFinishReason('pause_turn')).toBe('pause_turn');
+    });
+  });
+
+  describe('case normalization', () => {
+    it('should handle uppercase input', () => {
+      expect(normalizeFinishReason('STOP')).toBe('stop');
+      expect(normalizeFinishReason('LENGTH')).toBe('length');
+      expect(normalizeFinishReason('END_TURN')).toBe('stop');
+    });
+
+    it('should handle mixed case input', () => {
+      expect(normalizeFinishReason('Stop')).toBe('stop');
+      expect(normalizeFinishReason('Length')).toBe('length');
+      expect(normalizeFinishReason('End_Turn')).toBe('stop');
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle null and undefined', () => {
+      expect(normalizeFinishReason(null)).toBeUndefined();
+      expect(normalizeFinishReason(undefined)).toBeUndefined();
+    });
+
+    it('should handle empty and whitespace strings', () => {
+      expect(normalizeFinishReason('')).toBeUndefined();
+      expect(normalizeFinishReason('   ')).toBeUndefined();
+      expect(normalizeFinishReason('\t\n')).toBeUndefined();
+    });
+
+    it('should handle non-string input', () => {
+      expect(normalizeFinishReason(123 as any)).toBeUndefined();
+      expect(normalizeFinishReason({} as any)).toBeUndefined();
+      expect(normalizeFinishReason([] as any)).toBeUndefined();
+    });
+
+    it('should trim whitespace', () => {
+      expect(normalizeFinishReason('  stop  ')).toBe('stop');
+      expect(normalizeFinishReason('\tlength\n')).toBe('length');
+    });
+  });
+
+  describe('unmapped reasons', () => {
+    it('should pass through unknown reasons unchanged', () => {
+      expect(normalizeFinishReason('unknown_reason')).toBe('unknown_reason');
+      expect(normalizeFinishReason('custom_stop')).toBe('custom_stop');
+    });
+
+    it('should preserve case for unknown reasons after normalization', () => {
+      expect(normalizeFinishReason('CUSTOM_REASON')).toBe('custom_reason');
+    });
+  });
+});

test/util/formatDuration.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from 'vitest';
+import { formatDuration } from '../../src/util/formatDuration';
+
+describe('formatDuration', () => {
+  it('should format seconds only', () => {
+    expect(formatDuration(45)).toBe('45s');
+    expect(formatDuration(0)).toBe('0s');
+    expect(formatDuration(59)).toBe('59s');
+  });
+
+  it('should format minutes and seconds', () => {
+    expect(formatDuration(60)).toBe('1m 0s');
+    expect(formatDuration(65)).toBe('1m 5s');
+    expect(formatDuration(119)).toBe('1m 59s');
+    expect(formatDuration(600)).toBe('10m 0s');
+  });
+
+  it('should format hours, minutes, and seconds', () => {
+    expect(formatDuration(3600)).toBe('1h 0m 0s');
+    expect(formatDuration(3661)).toBe('1h 1m 1s');
+    expect(formatDuration(7382)).toBe('2h 3m 2s');
+  });
+
+  it('should handle edge cases correctly', () => {
+    // Test with decimal values - should round down
+    expect(formatDuration(45.9)).toBe('45s');
+
+    // Test with very large values
+    expect(formatDuration(86400)).toBe('24h 0m 0s'); // 1 day
+    expect(formatDuration(90061)).toBe('25h 1m 1s'); // 1 day, 1 hour, 1 minute, 1 second
+
+    // Test with string numbers (TypeScript should handle this implicitly)
+    expect(formatDuration(Number('120'))).toBe('2m 0s');
+  });
+});

test/util/functions/loadFunction.test.ts

@@ -0,0 +1,385 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { importModule } from '../../../src/esm';
+import { runPython } from '../../../src/python/pythonUtils';
+import {
+  functionCache,
+  loadFunction,
+  parseFileUrl,
+} from '../../../src/util/functions/loadFunction';
+
+vi.mock('../../../src/esm', () => ({
+  importModule: vi.fn(),
+}));
+
+vi.mock('../../../src/python/pythonUtils', () => ({
+  runPython: vi.fn(),
+}));
+
+// Use hoisted mock values that work on all platforms
+const { TEST_JS_PATH, TEST_PY_PATH, TEST_TXT_PATH, mockResolve } = vi.hoisted(() => {
+  const TEST_JS_PATH = '/test/resolved/function.js';
+  const TEST_PY_PATH = '/test/resolved/function.py';
+  const TEST_TXT_PATH = '/test/resolved/function.txt';
+  const mockResolve = vi.fn((...args: string[]): string => {
+    const lastArg = args[args.length - 1] || '';
+    if (lastArg.endsWith('.py')) {
+      return TEST_PY_PATH;
+    }
+    if (lastArg.endsWith('.txt')) {
+      return TEST_TXT_PATH;
+    }
+    return TEST_JS_PATH;
+  });
+  return { TEST_JS_PATH, TEST_PY_PATH, TEST_TXT_PATH, mockResolve };
+});
+
+vi.mock('path', async () => {
+  const actual = await vi.importActual<typeof import('path')>('path');
+  return {
+    ...actual,
+    default: {
+      ...actual,
+      resolve: mockResolve,
+    },
+    resolve: mockResolve,
+  };
+});
+
+vi.mock('../../../src/cliState', () => ({
+  default: {
+    basePath: '/base/path',
+  },
+}));
+
+describe('loadFunction', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockResolve.mockReset();
+    mockResolve.mockImplementation((...args: string[]) => {
+      const lastArg = args[args.length - 1] || '';
+      if (lastArg.endsWith('.py')) {
+        return TEST_PY_PATH;
+      }
+      if (lastArg.endsWith('.txt')) {
+        return TEST_TXT_PATH;
+      }
+      return TEST_JS_PATH;
+    });
+    // Clear the function cache
+    Object.keys(functionCache).forEach((key) => delete functionCache[key]);
+  });
+
+  describe('JavaScript functions', () => {
+    it('should load a JavaScript function with explicit function name', async () => {
+      const mockFn = vi.fn();
+      vi.mocked(importModule).mockResolvedValue(mockFn);
+
+      const result = await loadFunction({
+        filePath: 'function.js',
+        functionName: 'customFunction',
+      });
+
+      expect(importModule).toHaveBeenCalledWith(TEST_JS_PATH, 'customFunction');
+      expect(result).toBe(mockFn);
+    });
+
+    it('should load a JavaScript function with default export', async () => {
+      const mockFn = vi.fn();
+      vi.mocked(importModule).mockResolvedValue(mockFn);
+
+      const result = await loadFunction({
+        filePath: 'function.js',
+      });
+
+      expect(importModule).toHaveBeenCalledWith(TEST_JS_PATH, undefined);
+      expect(result).toBe(mockFn);
+    });
+
+    it('should load a JavaScript function from default.default export', async () => {
+      const mockFn = vi.fn();
+      vi.mocked(importModule).mockResolvedValue({ default: { default: mockFn } });
+
+      const result = await loadFunction({
+        filePath: 'function.js',
+      });
+
+      expect(importModule).toHaveBeenCalledWith(TEST_JS_PATH, undefined);
+      expect(result).toBe(mockFn);
+    });
+
+    it('should throw error if JavaScript file does not export a function', async () => {
+      vi.mocked(importModule).mockResolvedValue({ notAFunction: 'string' });
+
+      await expect(
+        loadFunction({
+          filePath: 'function.js',
+          functionName: 'customFunction',
+        }),
+      ).rejects.toThrow('JavaScript file must export a "customFunction" function');
+    });
+
+    it('should use function cache when enabled', async () => {
+      const mockFn = vi.fn();
+      vi.mocked(importModule).mockResolvedValue(mockFn);
+
+      // First call
+      const result1 = await loadFunction({
+        filePath: 'function.js',
+        useCache: true,
+      });
+
+      // Second call - should use cache
+      const result2 = await loadFunction({
+        filePath: 'function.js',
+        useCache: true,
+      });
+
+      expect(importModule).toHaveBeenCalledTimes(1);
+      expect(result1).toBe(result2);
+      expect(result1).toBe(mockFn);
+    });
+
+    it('should keep cached functions isolated by resolved path', async () => {
+      const mockFn1 = vi.fn();
+      const mockFn2 = vi.fn();
+      mockResolve.mockImplementation((basePath, filePath) => `${basePath}/${filePath}`);
+      vi.mocked(importModule).mockResolvedValueOnce(mockFn1).mockResolvedValueOnce(mockFn2);
+
+      const result1 = await loadFunction({
+        filePath: 'function.js',
+        basePath: '/base/one',
+        useCache: true,
+      });
+      const result2 = await loadFunction({
+        filePath: 'function.js',
+        basePath: '/base/two',
+        useCache: true,
+      });
+
+      expect(importModule).toHaveBeenCalledTimes(2);
+      expect(importModule).toHaveBeenNthCalledWith(1, '/base/one/function.js', undefined);
+      expect(importModule).toHaveBeenNthCalledWith(2, '/base/two/function.js', undefined);
+      expect(result1).toBe(mockFn1);
+      expect(result2).toBe(mockFn2);
+    });
+
+    it('should keep cached functions isolated by default function name', async () => {
+      const mockFn1 = vi.fn();
+      const mockFn2 = vi.fn();
+      vi.mocked(importModule).mockResolvedValue({ first: mockFn1, second: mockFn2 });
+
+      const result1 = await loadFunction({
+        filePath: 'function.js',
+        defaultFunctionName: 'first',
+        useCache: true,
+      });
+      const result2 = await loadFunction({
+        filePath: 'function.js',
+        defaultFunctionName: 'second',
+        useCache: true,
+      });
+
+      expect(importModule).toHaveBeenCalledTimes(2);
+      expect(result1).toBe(mockFn1);
+      expect(result2).toBe(mockFn2);
+    });
+
+    it('should not use cache when disabled', async () => {
+      const mockFn1 = vi.fn();
+      const mockFn2 = vi.fn();
+      vi.mocked(importModule).mockResolvedValueOnce(mockFn1).mockResolvedValueOnce(mockFn2);
+
+      // First call
+      const result1 = await loadFunction({
+        filePath: 'function.js',
+        useCache: false,
+      });
+
+      // Second call
+      const result2 = await loadFunction({
+        filePath: 'function.js',
+        useCache: false,
+      });
+
+      expect(importModule).toHaveBeenCalledTimes(2);
+      expect(result1).not.toBe(result2);
+    });
+  });
+
+  describe('Python functions', () => {
+    it('should load a Python function with explicit function name', async () => {
+      const mockPythonResult = vi.fn();
+      vi.mocked(runPython).mockImplementation((...args) => mockPythonResult(...args));
+
+      const result = await loadFunction({
+        filePath: 'function.py',
+        functionName: 'custom_function',
+      });
+
+      expect(typeof result).toBe('function');
+      await result('test input');
+      expect(runPython).toHaveBeenCalledWith(TEST_PY_PATH, 'custom_function', ['test input']);
+    });
+
+    it('should use default function name for Python when none specified', async () => {
+      const mockPythonResult = vi.fn();
+      vi.mocked(runPython).mockImplementation((...args) => mockPythonResult(...args));
+
+      const result = await loadFunction({
+        filePath: 'function.py',
+      });
+
+      expect(typeof result).toBe('function');
+      await result('test input');
+      expect(runPython).toHaveBeenCalledWith(TEST_PY_PATH, 'func', ['test input']);
+    });
+
+    it('should keep cached Python functions isolated by default function name', async () => {
+      const result1 = await loadFunction({
+        filePath: 'function.py',
+        defaultFunctionName: 'first',
+        useCache: true,
+      });
+      const result2 = await loadFunction({
+        filePath: 'function.py',
+        defaultFunctionName: 'second',
+        useCache: true,
+      });
+
+      expect(result1).not.toBe(result2);
+
+      await result1('first input');
+      await result2('second input');
+
+      expect(runPython).toHaveBeenNthCalledWith(1, TEST_PY_PATH, 'first', ['first input']);
+      expect(runPython).toHaveBeenNthCalledWith(2, TEST_PY_PATH, 'second', ['second input']);
+    });
+  });
+
+  describe('Error handling', () => {
+    it('should throw error for unsupported file types', async () => {
+      await expect(
+        loadFunction({
+          filePath: 'function.txt',
+        }),
+      ).rejects.toThrow(
+        'File must be a JavaScript (js, cjs, mjs, ts, cts, mts) or Python (.py) file',
+      );
+    });
+
+    it('should handle import errors', async () => {
+      const error = new Error('Import failed');
+      vi.mocked(importModule).mockRejectedValue(error);
+
+      await expect(
+        loadFunction({
+          filePath: 'function.js',
+        }),
+      ).rejects.toThrow('Import failed');
+    });
+  });
+});
+
+describe('parseFileUrl', () => {
+  it('should parse file URL with function name', () => {
+    const result = parseFileUrl('file:///path/to/file.js:functionName');
+    expect(result).toEqual({
+      filePath: '/path/to/file.js',
+      functionName: 'functionName',
+    });
+  });
+
+  it('should parse file URL without function name', () => {
+    const result = parseFileUrl('file:///path/to/file.js');
+    expect(result).toEqual({
+      filePath: '/path/to/file.js',
+    });
+  });
+
+  it('should throw error for invalid file URL', () => {
+    expect(() => parseFileUrl('/path/to/file.js')).toThrow('URL must start with file://');
+  });
+
+  it('should handle Windows-style paths', () => {
+    const result = parseFileUrl('file://C:/path/to/file.js:functionName');
+    expect(result).toEqual({
+      filePath: 'C:/path/to/file.js',
+      functionName: 'functionName',
+    });
+  });
+
+  it('should handle standard Windows file URLs on Windows', () => {
+    const originalPlatform = process.platform;
+    Object.defineProperty(process, 'platform', { value: 'win32', configurable: true });
+
+    try {
+      expect(parseFileUrl('file:///C:/path/to/file.js')).toEqual({
+        filePath: 'C:/path/to/file.js',
+      });
+      expect(parseFileUrl('file:///C:/path/to/file.js:functionName')).toEqual({
+        filePath: 'C:/path/to/file.js',
+        functionName: 'functionName',
+      });
+    } finally {
+      Object.defineProperty(process, 'platform', {
+        value: originalPlatform,
+        configurable: true,
+      });
+    }
+  });
+
+  it('should preserve standard Windows-looking file URLs as POSIX paths on POSIX', () => {
+    const originalPlatform = process.platform;
+    Object.defineProperty(process, 'platform', { value: 'linux', configurable: true });
+
+    try {
+      expect(parseFileUrl('file:///C:/path/to/file.js')).toEqual({
+        filePath: '/C:/path/to/file.js',
+      });
+    } finally {
+      Object.defineProperty(process, 'platform', {
+        value: originalPlatform,
+        configurable: true,
+      });
+    }
+  });
+
+  it('should handle relative paths', () => {
+    const result = parseFileUrl('file://./path/to/file.js:functionName');
+    expect(result).toEqual({
+      filePath: './path/to/file.js',
+      functionName: 'functionName',
+    });
+  });
+
+  it('should parse Python file URLs with function names', () => {
+    const result = parseFileUrl('file://./path/to/file.py:function_name');
+    expect(result).toEqual({
+      filePath: './path/to/file.py',
+      functionName: 'function_name',
+    });
+  });
+
+  it('should preserve colons in default-export file paths', () => {
+    const result = parseFileUrl('file://./path/to/file:default.js');
+    expect(result).toEqual({
+      filePath: './path/to/file:default.js',
+    });
+  });
+
+  it('should parse named exports from file paths that contain colons', () => {
+    const result = parseFileUrl('file://./path/to/file:default.js:functionName');
+    expect(result).toEqual({
+      filePath: './path/to/file:default.js',
+      functionName: 'functionName',
+    });
+  });
+
+  it('should parse named exports from directory paths that contain colons', () => {
+    const result = parseFileUrl('file://path:with:colons/hooks.js:functionName');
+    expect(result).toEqual({
+      filePath: 'path:with:colons/hooks.js',
+      functionName: 'functionName',
+    });
+  });
+});

test/util/generation.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, it, vi } from 'vitest';
+import { retryWithDeduplication, sampleArray } from '../../src/util/generation';
+
+describe('retryWithDeduplication', () => {
+  it('should collect unique items until target count is reached', async () => {
+    const operation = vi
+      .fn()
+      .mockResolvedValueOnce([1, 2, 3])
+      .mockResolvedValueOnce([3, 4, 5])
+      .mockResolvedValueOnce([5, 6, 7]);
+
+    const result = await retryWithDeduplication(operation, 5);
+
+    expect(result).toEqual([1, 2, 3, 4, 5]);
+    expect(operation).toHaveBeenCalledTimes(2);
+  });
+
+  it('should stop after max consecutive retries', async () => {
+    const operation = vi
+      .fn()
+      .mockResolvedValueOnce([1, 2])
+      .mockResolvedValueOnce([1, 2])
+      .mockResolvedValueOnce([1, 2]);
+
+    const result = await retryWithDeduplication(operation, 5, 2);
+
+    expect(result).toEqual([1, 2]);
+    expect(operation).toHaveBeenCalledTimes(4);
+  });
+
+  it('should use custom deduplication function', async () => {
+    const operation = vi
+      .fn()
+      .mockResolvedValueOnce([{ id: 1 }, { id: 2 }])
+      .mockResolvedValueOnce([{ id: 2 }, { id: 3 }]);
+
+    const customDedupFn = (items: { id: number }[]) =>
+      Array.from(new Set(items.map((item) => item.id))).map((id) => ({ id }));
+
+    const result = await retryWithDeduplication(operation, 3, 2, customDedupFn);
+
+    expect(result).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]);
+    expect(operation).toHaveBeenCalledTimes(2);
+  });
+
+  it('should handle empty results from operation', async () => {
+    const operation = vi
+      .fn()
+      .mockResolvedValueOnce([1, 2])
+      .mockResolvedValueOnce([])
+      .mockResolvedValueOnce([3]);
+
+    const result = await retryWithDeduplication(operation, 3);
+
+    expect(result).toEqual([1, 2, 3]);
+    expect(operation).toHaveBeenCalledTimes(3);
+  });
+
+  it('should return all unique items even if target count is not reached', async () => {
+    const operation = vi
+      .fn()
+      .mockResolvedValueOnce([1, 2])
+      .mockResolvedValueOnce([2, 3])
+      .mockResolvedValueOnce([3, 4]);
+
+    const result = await retryWithDeduplication(operation, 10, 2);
+
+    expect(result).toEqual([1, 2, 3, 4]);
+    expect(operation).toHaveBeenCalledTimes(6);
+  });
+});
+
+describe('sampleArray', () => {
+  it('should return n random items when n is less than array length', () => {
+    const array = [1, 2, 3, 4, 5];
+    const result = sampleArray(array, 3);
+
+    expect(result).toHaveLength(3);
+    expect(new Set(result).size).toBe(3); // All items are unique
+    result.forEach((item) => expect(array).toContain(item));
+  });
+
+  it('should return all items when n is equal to array length', () => {
+    const array = [1, 2, 3, 4, 5];
+    const result = sampleArray(array, 5);
+
+    expect(result).toHaveLength(5);
+    expect(new Set(result).size).toBe(5);
+    expect(result).toEqual(expect.arrayContaining(array));
+  });
+
+  it('should return all items when n is greater than array length', () => {
+    const array = [1, 2, 3];
+    const result = sampleArray(array, 5);
+
+    expect(result).toHaveLength(3);
+    expect(result).toEqual(expect.arrayContaining(array));
+  });
+
+  it('should return an empty array when input array is empty', () => {
+    const result = sampleArray([], 3);
+    expect(result).toEqual([]);
+  });
+
+  it('should return a new array, not modifying the original', () => {
+    const array = [1, 2, 3, 4, 5];
+    const originalArray = [...array];
+    sampleArray(array, 3);
+
+    expect(array).toEqual(originalArray);
+  });
+
+  it('should return random samples across multiple calls', () => {
+    const array = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+    const samples = new Set();
+
+    for (let i = 0; i < 100; i++) {
+      const result = sampleArray(array, 5);
+      samples.add(result.join(','));
+    }
+
+    // With 100 samples, it's extremely unlikely to get the same sample every time
+    // unless the randomization is not working
+    expect(samples.size).toBeGreaterThan(1);
+  });
+});

test/util/gradingProvider.test.ts

@@ -0,0 +1,170 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+  buildConfiguredProviderMap,
+  GRADING_PROVIDER_TYPE_KEYS,
+  isProviderTypeMap,
+  resolveConfiguredProviderReference,
+} from '../../src/util/gradingProvider';
+
+import type { ApiProvider } from '../../src/types/providers';
+
+function makeProvider(id: string, label?: string): ApiProvider {
+  return {
+    id: vi.fn().mockReturnValue(id),
+    label,
+    callApi: vi.fn().mockResolvedValue({ output: '' }),
+  };
+}
+
+describe('GRADING_PROVIDER_TYPE_KEYS', () => {
+  it('lists the four supported grading-provider types', () => {
+    expect([...GRADING_PROVIDER_TYPE_KEYS]).toEqual([
+      'text',
+      'embedding',
+      'classification',
+      'moderation',
+    ]);
+  });
+});
+
+describe('isProviderTypeMap', () => {
+  it.each([
+    ['null', null, false],
+    ['undefined', undefined, false],
+    ['string', 'litellm:judge', false],
+    ['array', ['litellm:judge'], false],
+    ['empty object', {}, false],
+    ['ProviderOptions (has id)', { id: 'litellm:judge', config: {} }, false],
+    [
+      'ApiProvider instance',
+      { id: () => 'x', callApi: () => Promise.resolve({ output: '' }) },
+      false,
+    ],
+    ['ProviderOptionsMap (unknown key)', { 'litellm:judge': { config: {} } }, false],
+    ['ProviderOptionsMap (type-named key)', { text: { config: {} } }, false],
+    ['type key with empty string value', { text: '' }, false],
+    ['type key with undefined value', { text: undefined }, false],
+  ])('returns false for %s', (_label, value, expected) => {
+    expect(isProviderTypeMap(value)).toBe(expected);
+  });
+
+  it.each([
+    ['text', { text: 'litellm:judge' }],
+    ['text ProviderOptions', { text: { id: 'litellm:judge', config: {} } }],
+    ['embedding', { embedding: 'litellm:embed' }],
+    ['classification', { classification: 'classifier' }],
+    ['moderation', { moderation: 'moderator' }],
+    ['multiple', { text: 'litellm:judge', embedding: 'litellm:embed' }],
+  ])('returns true for typed map (%s)', (_label, value) => {
+    expect(isProviderTypeMap(value)).toBe(true);
+  });
+});
+
+describe('buildConfiguredProviderMap', () => {
+  it('returns a null-prototype map', () => {
+    const map = buildConfiguredProviderMap([makeProvider('a')]);
+    expect(Object.getPrototypeOf(map)).toBeNull();
+  });
+
+  it('indexes providers by id and label', () => {
+    const provider = makeProvider('openai:gpt-4', 'gpt');
+    const map = buildConfiguredProviderMap([provider]);
+    expect(map['openai:gpt-4']).toBe(provider);
+    expect(map.gpt).toBe(provider);
+  });
+
+  it('does not let a later provider label shadow an earlier provider id', () => {
+    const byId = makeProvider('judge');
+    const byLabel = makeProvider('litellm:judge', 'judge');
+    const map = buildConfiguredProviderMap([byId, byLabel]);
+    expect(map.judge).toBe(byId);
+    expect(map['litellm:judge']).toBe(byLabel);
+  });
+
+  it('is independent of provider iteration order for id/label collisions', () => {
+    const byId = makeProvider('judge');
+    const byLabel = makeProvider('litellm:judge', 'judge');
+    const reverse = buildConfiguredProviderMap([byLabel, byId]);
+    expect(reverse.judge).toBe(byId);
+    expect(reverse['litellm:judge']).toBe(byLabel);
+  });
+
+  it('still applies a label alias when no id collides', () => {
+    const provider = makeProvider('litellm:judge', 'judge');
+    const map = buildConfiguredProviderMap([provider]);
+    expect(map.judge).toBe(provider);
+    expect(map['litellm:judge']).toBe(provider);
+  });
+
+  it('does not surface prototype properties through hasOwn lookups', () => {
+    const map = buildConfiguredProviderMap([makeProvider('a')]);
+    expect(Object.hasOwn(map, '__proto__')).toBe(false);
+    expect(Object.hasOwn(map, 'constructor')).toBe(false);
+    expect(Object.hasOwn(map, 'toString')).toBe(false);
+  });
+
+  it('returns an empty null-prototype map for an empty input', () => {
+    const map = buildConfiguredProviderMap([]);
+    expect(Object.getPrototypeOf(map)).toBeNull();
+    expect(Object.keys(map)).toEqual([]);
+  });
+});
+
+describe('resolveConfiguredProviderReference', () => {
+  it('preserves a fully resolved provider map when no entry needs env injection', () => {
+    const provider = makeProvider('litellm:judge');
+    const typeMap = { text: provider };
+
+    expect(
+      resolveConfiguredProviderReference(typeMap, buildConfiguredProviderMap([]), {
+        API_KEY: 'suite-key',
+      }),
+    ).toBe(typeMap);
+  });
+
+  it('adds env only when a deferred typed provider must still be loaded', () => {
+    const embeddingProvider = makeProvider('litellm:embedding:judge');
+    const typeMap = { text: 'litellm:judge', embedding: embeddingProvider };
+
+    expect(
+      resolveConfiguredProviderReference(typeMap, buildConfiguredProviderMap([]), {
+        API_KEY: 'suite-key',
+      }),
+    ).toEqual({
+      text: { id: 'litellm:judge', env: { API_KEY: 'suite-key' } },
+      embedding: embeddingProvider,
+    });
+  });
+
+  it('resolves configured typed entries while leaving unconfigured alternatives lazy', () => {
+    const textProvider = makeProvider('litellm:judge');
+    const providerMap = buildConfiguredProviderMap([textProvider]);
+
+    expect(
+      resolveConfiguredProviderReference(
+        { text: 'litellm:judge', embedding: 'unsupported-provider:unused-embedding' },
+        providerMap,
+      ),
+    ).toEqual({
+      text: textProvider,
+      embedding: 'unsupported-provider:unused-embedding',
+    });
+  });
+
+  it('resolves an id-only typed provider option to a configured provider instance', () => {
+    const textProvider = makeProvider('litellm:judge');
+    const providerMap = buildConfiguredProviderMap([textProvider]);
+
+    expect(
+      resolveConfiguredProviderReference({ text: { id: 'litellm:judge' } }, providerMap),
+    ).toEqual({ text: textProvider });
+  });
+
+  it('keeps a typed provider option with overrides inline', () => {
+    const textProvider = makeProvider('litellm:judge');
+    const providerMap = buildConfiguredProviderMap([textProvider]);
+    const inlineProvider = { text: { id: 'litellm:judge', config: { temperature: 0 } } };
+
+    expect(resolveConfiguredProviderReference(inlineProvider, providerMap)).toBe(inlineProvider);
+  });
+});

test/util/inlineBlobsForShare.test.ts

@@ -0,0 +1,70 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { getShareAuthorizedBlob } from '../../src/blobs';
+import { createBlobInlineCache, inlineBlobRefsForShare } from '../../src/util/inlineBlobsForShare';
+
+import type { StoredBlob } from '../../src/blobs';
+
+vi.mock('../../src/blobs', () => ({
+  getShareAuthorizedBlob: vi.fn(),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    warn: vi.fn(),
+  },
+}));
+
+describe('inlineBlobRefsForShare', () => {
+  const hash = 'a'.repeat(64);
+  const uri = `promptfoo://blob/${hash}`;
+  const bytes = Buffer.from('image-bytes');
+  const storedBlob: StoredBlob = {
+    data: bytes,
+    metadata: {
+      createdAt: '2026-06-08T00:00:00.000Z',
+      key: 'blob-key',
+      mimeType: 'image/png',
+      provider: 'filesystem',
+      sizeBytes: bytes.length,
+    },
+  };
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(getShareAuthorizedBlob).mockResolvedValue(storedBlob);
+  });
+
+  it('inlines authorized blob URIs as data URLs', async () => {
+    const result = await inlineBlobRefsForShare(
+      { output: uri },
+      createBlobInlineCache(),
+      'local-eval-1',
+    );
+
+    expect(result.output).toBe(`data:image/png;base64,${bytes.toString('base64')}`);
+    expect(getShareAuthorizedBlob).toHaveBeenCalledWith(hash, 'local-eval-1');
+  });
+
+  it('does not inline a copied blob URI that is not share-authorized for the eval', async () => {
+    vi.mocked(getShareAuthorizedBlob).mockResolvedValue(null);
+
+    const result = await inlineBlobRefsForShare(
+      { output: uri },
+      createBlobInlineCache(),
+      'local-eval-1',
+    );
+
+    expect(result.output).toBe(uri);
+    expect(getShareAuthorizedBlob).toHaveBeenCalledWith(hash, 'local-eval-1');
+  });
+
+  it('caches the authorization decision across calls sharing a cache', async () => {
+    vi.mocked(getShareAuthorizedBlob).mockResolvedValue(null);
+    const cache = createBlobInlineCache();
+
+    await inlineBlobRefsForShare({ output: uri }, cache, 'local-eval-1');
+    await inlineBlobRefsForShare([uri], cache, 'local-eval-1');
+
+    expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
+  });
+});

test/util/invariant.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest';
+import invariant from '../../src/util/invariant';
+
+describe('invariant', () => {
+  it('should not throw when condition is true', () => {
+    expect(() => invariant(true)).not.toThrow();
+    expect(() => invariant(1)).not.toThrow();
+    expect(() => invariant({})).not.toThrow();
+  });
+
+  it('should throw when condition is false', () => {
+    expect(() => invariant(false)).toThrow('Invariant failed');
+    expect(() => invariant(0)).toThrow('Invariant failed');
+    expect(() => invariant(null)).toThrow('Invariant failed');
+    expect(() => invariant(undefined)).toThrow('Invariant failed');
+  });
+
+  it('should include the provided message in the error', () => {
+    const message = 'Custom error message';
+    expect(() => invariant(false, message)).toThrow('Invariant failed: Custom error message');
+  });
+
+  it('should support message callback functions', () => {
+    const getMessage = () => 'Dynamic message';
+    expect(() => invariant(false, getMessage)).toThrow('Invariant failed: Dynamic message');
+  });
+
+  it('should work for type narrowing', () => {
+    const getValue = (): string | null => 'test';
+    const value = getValue();
+
+    // This should compile without type errors
+    invariant(value !== null, 'Value should not be null');
+
+    // After the invariant, TypeScript should know that value is a string
+    const length: number = value.length;
+    expect(length).toBe(4);
+  });
+});

test/util/isPathWithinDir.test.ts

@@ -0,0 +1,92 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { isPathWithinDir } from '../../src/util/isPathWithinDir';
+
+describe('isPathWithinDir', () => {
+  let testRoot: string;
+  let workspace: string;
+  let outsideDir: string;
+
+  beforeEach(async () => {
+    testRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'permissions-test-'));
+    workspace = path.join(testRoot, 'workspace');
+    outsideDir = path.join(testRoot, 'outside');
+
+    await fs.mkdir(workspace);
+    await fs.mkdir(outsideDir);
+    await fs.mkdir(path.join(workspace, 'subdir'));
+
+    await fs.writeFile(path.join(workspace, 'file.txt'), 'test');
+    await fs.writeFile(path.join(workspace, 'subdir', 'nested.txt'), 'test');
+    await fs.writeFile(path.join(outsideDir, 'external.txt'), 'test');
+
+    // Create a symlink that points outside the workspace (for symlink attack testing)
+    await fs.symlink(path.join(outsideDir, 'external.txt'), path.join(workspace, 'evil-symlink'));
+  });
+
+  afterEach(async () => {
+    await fs.rm(testRoot, { recursive: true, force: true });
+  });
+
+  it('should allow files within directory', async () => {
+    await expect(isPathWithinDir('file.txt', workspace)).resolves.toBe(true);
+    await expect(isPathWithinDir('./file.txt', workspace)).resolves.toBe(true);
+    await expect(isPathWithinDir('subdir/nested.txt', workspace)).resolves.toBe(true);
+    await expect(isPathWithinDir('./subdir/nested.txt', workspace)).resolves.toBe(true);
+    await expect(isPathWithinDir(path.join(workspace, 'file.txt'), workspace)).resolves.toBe(true);
+  });
+
+  it('should block files outside directory using relative paths', async () => {
+    await expect(isPathWithinDir('../outside/external.txt', workspace)).resolves.toBe(false);
+    await expect(isPathWithinDir('../../external.txt', workspace)).resolves.toBe(false);
+  });
+
+  it('should block files outside directory using absolute paths', async () => {
+    await expect(isPathWithinDir('/etc/hosts', workspace)).resolves.toBe(false);
+    await expect(isPathWithinDir(path.join(outsideDir, 'external.txt'), workspace)).resolves.toBe(
+      false,
+    );
+  });
+
+  it('should block access to parent directory', async () => {
+    await expect(isPathWithinDir('..', workspace)).resolves.toBe(false);
+    await expect(isPathWithinDir('../', workspace)).resolves.toBe(false);
+  });
+
+  it('should allow directory itself', async () => {
+    await expect(isPathWithinDir('.', workspace)).resolves.toBe(true);
+    await expect(isPathWithinDir(workspace, workspace)).resolves.toBe(true);
+  });
+
+  it('should handle path traversal attempts', async () => {
+    await expect(isPathWithinDir('subdir/../../outside/external.txt', workspace)).resolves.toBe(
+      false,
+    );
+  });
+
+  it('should block symlinks that point outside directory (symlink attack)', async () => {
+    await expect(isPathWithinDir('evil-symlink', workspace)).resolves.toBe(false);
+  });
+
+  it('should allow non-existent paths within directory (for Write tool)', async () => {
+    await expect(isPathWithinDir('nonexistent.txt', workspace)).resolves.toBe(true);
+    await expect(isPathWithinDir('subdir/new-file.txt', workspace)).resolves.toBe(true);
+
+    await expect(isPathWithinDir('../nonexistent.txt', workspace)).resolves.toBe(false);
+    await expect(isPathWithinDir('../../nonexistent.txt', workspace)).resolves.toBe(false);
+  });
+
+  it('should throw error immediately if directory itself does not exist', async () => {
+    const nonExistentDir = path.join(testRoot, 'does-not-exist');
+
+    await expect(isPathWithinDir('file.txt', nonExistentDir)).rejects.toThrow(
+      'Directory does not exist or is inaccessible',
+    );
+    await expect(isPathWithinDir('subdir/file.txt', nonExistentDir)).rejects.toThrow(
+      'Directory does not exist or is inaccessible',
+    );
+  });
+});

test/util/json.test.ts

@@ -0,0 +1,828 @@
+import dedent from 'dedent';
+import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { ResultFailureReason } from '../../src/types/index';
+import {
+  convertSlashCommentsToHash,
+  extractFirstJsonObject,
+  extractJsonObjects,
+  getAjv,
+  isValidJson,
+  orderKeys,
+  resetAjv,
+  safeJsonStringify,
+  summarizeEvaluateResultForLogging,
+} from '../../src/util/json';
+import { createEvaluateResult } from '../factories/eval';
+import { createAtomicTestCase, createPrompt } from '../factories/testSuite';
+import { mockProcessEnv } from './utils';
+
+import type { EvaluateResult } from '../../src/types/index';
+
+describe('json utilities', () => {
+  describe('getAjv', () => {
+    beforeAll(() => {
+      mockProcessEnv({ NODE_ENV: 'test' });
+    });
+
+    beforeEach(() => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_AJV_STRICT_MODE: undefined });
+      resetAjv();
+    });
+
+    afterEach(() => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_AJV_STRICT_MODE: undefined });
+    });
+
+    it('should create an Ajv instance with default options', () => {
+      const ajv = getAjv();
+      expect(ajv).toBeDefined();
+      expect(ajv.opts.strictSchema).toBe(true);
+    });
+
+    it('should disable strict mode when PROMPTFOO_DISABLE_AJV_STRICT_MODE is set', () => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_AJV_STRICT_MODE: 'true' });
+      const ajv = getAjv();
+      expect(ajv.opts.strictSchema).toBe(false);
+    });
+
+    it('should add formats to the Ajv instance', () => {
+      const ajv = getAjv();
+      expect(ajv.formats).toBeDefined();
+      expect(Object.keys(ajv.formats)).not.toHaveLength(0);
+    });
+
+    it('should reuse the same instance on subsequent calls', () => {
+      const firstInstance = getAjv();
+      const secondInstance = getAjv();
+      expect(firstInstance).toBe(secondInstance);
+    });
+
+    it('should reset the instance when resetAjv is called', () => {
+      const firstInstance = getAjv();
+      resetAjv();
+      const secondInstance = getAjv();
+      expect(firstInstance).not.toBe(secondInstance);
+    });
+
+    it('should only allow resetAjv to be called in test environment', () => {
+      const originalNodeEnv = process.env.NODE_ENV;
+      try {
+        mockProcessEnv({ NODE_ENV: 'production' });
+        expect(() => resetAjv()).toThrow('resetAjv can only be called in test environment');
+      } finally {
+        mockProcessEnv({ NODE_ENV: originalNodeEnv });
+      }
+    });
+  });
+
+  describe('isValidJson', () => {
+    it('returns true for valid JSON', () => {
+      expect(isValidJson('{"key": "value"}')).toBe(true);
+      expect(isValidJson('[1, 2, 3]')).toBe(true);
+      expect(isValidJson('"string"')).toBe(true);
+      expect(isValidJson('123')).toBe(true);
+      expect(isValidJson('true')).toBe(true);
+      expect(isValidJson('null')).toBe(true);
+    });
+
+    it('returns false for invalid JSON', () => {
+      expect(isValidJson('{')).toBe(false);
+      expect(isValidJson('["unclosed array"')).toBe(false);
+      expect(isValidJson('{"key": value}')).toBe(false);
+      expect(isValidJson('undefined')).toBe(false);
+    });
+  });
+
+  describe('safeJsonStringify', () => {
+    it('stringifies simple objects', () => {
+      const obj: { key: string; number: number } = { key: 'value', number: 123 };
+      expect(safeJsonStringify(obj)).toBe('{"key":"value","number":123}');
+    });
+
+    it('handles circular references', () => {
+      const obj: { key: string; circular?: any } = { key: 'value' };
+      obj.circular = obj;
+      expect(safeJsonStringify(obj)).toBe('{"key":"value"}');
+    });
+
+    it('pretty prints when specified', () => {
+      const obj: { key: string; nested: { inner: string } } = {
+        key: 'value',
+        nested: { inner: 'content' },
+      };
+      const expected = dedent`
+      {
+        "key": "value",
+        "nested": {
+          "inner": "content"
+        }
+      }`;
+      expect(safeJsonStringify(obj, true)).toBe(expected);
+    });
+
+    it('handles null values', () => {
+      expect(safeJsonStringify(null)).toBe('null');
+    });
+
+    it('handles undefined values', () => {
+      expect(safeJsonStringify(undefined)).toBeUndefined();
+    });
+
+    it('returns undefined or strips non-serializable values', () => {
+      expect(safeJsonStringify(() => {})).toBeUndefined(); // Function
+      expect(safeJsonStringify(Symbol('sym'))).toBeUndefined(); // Symbol
+      expect(safeJsonStringify({ key: 'value' })).toBe('{"key":"value"}');
+    });
+
+    it('returns undefined for circular references in arrays', () => {
+      const arr: (number | any[])[] = [1, 2, 3];
+      arr.push(arr);
+      expect(safeJsonStringify(arr)).toBe('[1,2,3,null]');
+    });
+
+    it('handles complex nested structures', () => {
+      const complex: {
+        string: string;
+        number: number;
+        boolean: boolean;
+        null: null;
+        array: (number | string | { three: number })[];
+        nested: { a: number; b: number[] };
+      } = {
+        string: 'value',
+        number: 123,
+        boolean: true,
+        null: null,
+        array: [1, 'two', { three: 3 }],
+        nested: {
+          a: 1,
+          b: [2, 3],
+        },
+      };
+      const result = safeJsonStringify(complex);
+      expect(result).toBeDefined(); // Ensure it returns a string
+      expect(JSON.parse(result as string)).toEqual(complex); // Ensure it matches the original structure
+    });
+
+    it('handles arrays with circular references', () => {
+      const arr: any[] = [1, 2, 3];
+      arr.push(arr);
+      expect(safeJsonStringify(arr)).toBe('[1,2,3,null]');
+    });
+
+    it('handles nested circular references', () => {
+      const obj: any = { a: { b: {} } };
+      obj.a.b.c = obj.a;
+      expect(safeJsonStringify(obj)).toBe('{"a":{"b":{}}}');
+    });
+
+    it('preserves shared non-circular references', () => {
+      const shared = { value: 'shared' };
+      expect(safeJsonStringify({ a: shared, b: shared, c: [shared] })).toBe(
+        '{"a":{"value":"shared"},"b":{"value":"shared"},"c":[{"value":"shared"}]}',
+      );
+    });
+
+    it('preserves non-circular nested structures', () => {
+      const nested = { a: { b: { c: 1 } }, d: [1, 2, { e: 3 }] };
+      expect(JSON.parse(safeJsonStringify(nested) as string)).toEqual(nested);
+    });
+  });
+
+  describe('extractJsonObjects', () => {
+    it('should extract a single JSON object from a string', () => {
+      const input = '{"key": "value"}';
+      const expectedOutput = [{ key: 'value' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should extract multiple JSON objects from a string', () => {
+      const input = 'yolo {"key1": "value1"} some text {"key2": "value2"} fomo';
+      const expectedOutput = [{ key1: 'value1' }, { key2: 'value2' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should return an empty array if no JSON objects are found', () => {
+      const input = 'no json here';
+      const expectedOutput: any[] = [];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle nested JSON objects', () => {
+      const input = 'wassup {"outer": {"inner": "value"}, "foo": [1,2,3,4]}';
+      const expectedOutput = [{ outer: { inner: 'value' }, foo: [1, 2, 3, 4] }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle comments', () => {
+      const input = `{
+        "reason": "all good",
+        "score": 1.0 // perfect score
+      }`;
+      const expectedOutput = [{ reason: 'all good', score: 1.0 }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle jsonl', () => {
+      const input = `{"reason": "hello", "score": 1.0}
+      {"reason": "world", "score": 0.0}`;
+      const expectedOutput = [
+        { reason: 'hello', score: 1.0 },
+        { reason: 'world', score: 0.0 },
+      ];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle invalid JSON gracefully', () => {
+      const input = '{"key": "value" some text {"key2": "value2"}';
+      const expectedOutput = [{ key2: 'value2' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle incomplete JSON', () => {
+      const input = `{
+  "incomplete": "object"`;
+      expect(extractJsonObjects(input)).toEqual([
+        {
+          incomplete: 'object',
+        },
+      ]);
+    });
+
+    it('should handle string containing incomplete JSON', () => {
+      const input = dedent`{
+        "key1": "value1",
+        "key2": {
+          "nested": "value2"
+        },
+        "key3": "value3"
+      }
+      {
+        "incomplete": "object"`;
+      expect(extractJsonObjects(input)).toEqual([
+        {
+          key1: 'value1',
+          key2: {
+            nested: 'value2',
+          },
+          key3: 'value3',
+        },
+        {
+          incomplete: 'object',
+        },
+      ]);
+    });
+
+    it('should handle this case', () => {
+      const obj = {
+        vars: [
+          {
+            language: 'Klingon',
+            body: 'Live long and prosper',
+          },
+          {
+            language: 'Elvish',
+            body: 'Good morning',
+          },
+          {
+            language: 'Esperanto',
+            body: 'I love learning languages',
+          },
+          {
+            language: 'Morse Code',
+            body: 'Help',
+          },
+          {
+            language: 'Emoji',
+            body: 'I am feeling happy 😊',
+          },
+          {
+            language: 'Binary',
+            body: 'Yes',
+          },
+          {
+            language: 'Javascript',
+            body: 'Hello, World!',
+          },
+          {
+            language: 'Shakespearean',
+            body: 'To be or not to be',
+          },
+          {
+            language: 'Leet Speak',
+            body: 'You are amazing',
+          },
+          {
+            language: 'Old English',
+            body: 'What is thy name?',
+          },
+          {
+            language: 'Yoda Speak',
+            body: 'Strong with the force, you are',
+          },
+        ],
+      };
+      const input = JSON.stringify(obj);
+      expect(extractJsonObjects(input)).toEqual([obj]);
+    });
+
+    it('should handle YAML-style unquoted strings', () => {
+      const input = '{key: value, another_key: another value}';
+      const expectedOutput = [{ key: 'value', another_key: 'another value' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    describe('convertSlashCommentsToHash', () => {
+      it('should convert basic // comments to # comments', () => {
+        const input = 'some text // this is a comment';
+        const expected = 'some text # this is a comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should not convert // inside double quoted strings', () => {
+        const input = '"this // is not a comment" // but this is';
+        const expected = '"this // is not a comment" # but this is';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should not convert // inside single quoted strings', () => {
+        const input = "'don't convert // here' // convert here";
+        const expected = "'don't convert // here' # convert here";
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle escaped quotes in strings', () => {
+        const input = '"escaped \\" quote // not a comment" // real comment';
+        const expected = '"escaped \\" quote // not a comment" # real comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle multiple comments in one line', () => {
+        const input = 'text // first comment // not a second comment';
+        const expected = 'text # first comment // not a second comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle comments at the start of the line', () => {
+        const input = '// comment at start\ntext // comment at end';
+        const expected = '# comment at start\ntext # comment at end';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle strings with multiple lines', () => {
+        const input = dedent`
+          line1 // comment 1
+          "string with // not a comment"
+          line3 // comment 2`;
+        const expected = dedent`
+          line1 # comment 1
+          "string with // not a comment"
+          line3 # comment 2`;
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle nested quotes', () => {
+        const input = '"outer \\"inner // not comment\\" still outer" // real comment';
+        const expected = '"outer \\"inner // not comment\\" still outer" # real comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle mixed single and double quotes', () => {
+        const input = '"contains \'nested\' // not comment" // real comment';
+        const expected = '"contains \'nested\' // not comment" # real comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle empty strings', () => {
+        expect(convertSlashCommentsToHash('')).toBe('');
+      });
+
+      it('should handle strings without comments', () => {
+        const input = 'no comments here';
+        expect(convertSlashCommentsToHash(input)).toBe(input);
+      });
+
+      it('should handle strings with only a comment', () => {
+        const input = '// just a comment';
+        const expected = '# just a comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle multiple sequential comment markers', () => {
+        const input = 'text //// double comment';
+        const expected = 'text ## double comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle JSON-like structures', () => {
+        const input =
+          '{\n  "key": "value", // comment here\n  "key2": "value2" // another comment\n}';
+        const expected =
+          '{\n  "key": "value", # comment here\n  "key2": "value2" # another comment\n}';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+
+      it('should handle comments after different types of values', () => {
+        const inputs = [
+          'null // after null',
+          'true // after boolean',
+          '42 // after number',
+          '"string" // after string',
+          '[] // after array',
+          '{} // after object',
+        ];
+        const expected = inputs.map((input) => input.replace('//', '#'));
+        inputs.forEach((input, i) => {
+          expect(convertSlashCommentsToHash(input)).toBe(expected[i]);
+        });
+      });
+
+      it('should not treat URL schemes as comments', () => {
+        const inputs = [
+          'url: http://example.com/path',
+          'url: https://example.com/path',
+          'url: http://example.com//double-slash',
+        ];
+        inputs.forEach((input) => {
+          expect(convertSlashCommentsToHash(input)).toBe(input);
+        });
+      });
+
+      it('should still convert comments after URLs', () => {
+        const input = 'url: http://example.com/path // trailing comment';
+        const expected = 'url: http://example.com/path # trailing comment';
+        expect(convertSlashCommentsToHash(input)).toBe(expected);
+      });
+    });
+
+    it('should skip non-object YAML values', () => {
+      const input = 'plain string {key: value} 42 {another: obj} true';
+      const expectedOutput = [{ key: 'value' }, { another: 'obj' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle YAML with mixed quoted and unquoted strings', () => {
+      const input = '{unquoted: value, "quoted": "value", another: "mixed"}';
+      const expectedOutput = [{ unquoted: 'value', quoted: 'value', another: 'mixed' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should ignore YAML arrays and scalars', () => {
+      const input = '[1, 2, 3] {obj: value} plain: string {another: obj}';
+      const expectedOutput = [{ obj: 'value' }, { another: 'obj' }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    it('should handle YAML with special characters', () => {
+      const input = '{key: value with spaces, special: value!@#$%, empty:}';
+      const expectedOutput = [{ key: 'value with spaces', special: 'value!@#$%', empty: null }];
+      expect(extractJsonObjects(input)).toEqual(expectedOutput);
+    });
+
+    describe('LLM output scenarios', () => {
+      it('should handle JSON with inline comments', () => {
+        const input = dedent`
+          {
+            "reason": "The test passed",
+            "score": 1.0 # The score is perfect because all criteria were met
+          }`;
+        const expectedOutput = [
+          {
+            reason: 'The test passed',
+            score: 1.0,
+          },
+        ];
+        expect(extractJsonObjects(input)).toEqual(expectedOutput);
+      });
+
+      it('should handle JSON with multiline string containing markdown', () => {
+        const input = dedent`
+          {
+            "reason": "The summary provided is mostly accurate but contains some omissions:
+
+            - Point 1: Details here
+            - Point 2: More details
+
+            Some additional context...",
+            "score": 0.75
+          }`;
+        const expectedOutput = [
+          {
+            reason:
+              'The summary provided is mostly accurate but contains some omissions:\n- Point 1: Details here - Point 2: More details\nSome additional context...',
+            score: 0.75,
+          },
+        ];
+        expect(extractJsonObjects(input)).toEqual(expectedOutput);
+      });
+
+      it('should handle JSON with unescaped backticks', () => {
+        const input = dedent`
+          {
+            "reason": "\`\`The summary accurately reflects...",
+            "score": 1.0
+          }`;
+        const expectedOutput = [
+          {
+            reason: '``The summary accurately reflects...',
+            score: 1.0,
+          },
+        ];
+        expect(extractJsonObjects(input)).toEqual(expectedOutput);
+      });
+
+      it('should handle JSON with trailing commas', () => {
+        const input = dedent`
+          {
+            "key1": "value1",
+            "key2": "value2",
+          }`;
+        const expectedOutput = [
+          {
+            key1: 'value1',
+            key2: 'value2',
+          },
+        ];
+        expect(extractJsonObjects(input)).toEqual(expectedOutput);
+      });
+
+      it('should handle multiple JSON objects in LLM output with text in between', () => {
+        const input = dedent`
+          Here's my analysis:
+          {
+            "first_point": "valid observation",
+            "score": 0.8
+          }
+
+          And another point:
+          {
+            "second_point": "another observation",
+            "score": 0.9
+          }`;
+        const expectedOutput = [
+          {
+            first_point: 'valid observation',
+            score: 0.8,
+          },
+          {
+            second_point: 'another observation',
+            score: 0.9,
+          },
+        ];
+        expect(extractJsonObjects(input)).toEqual(expectedOutput);
+      });
+    });
+  });
+
+  describe('orderKeys', () => {
+    it('orders keys according to specified order', () => {
+      const obj = { c: 3, a: 1, b: 2 };
+      const order: (keyof typeof obj)[] = ['a', 'b', 'c'];
+      const result = orderKeys(obj, order);
+      expect(Object.keys(result)).toEqual(['a', 'b', 'c']);
+    });
+
+    it('places unspecified keys at the end', () => {
+      const obj = { d: 4, b: 2, a: 1, c: 3 };
+      const order: (keyof typeof obj)[] = ['a', 'b'];
+      const result = orderKeys(obj, order);
+      expect(Object.keys(result)).toEqual(['a', 'b', 'd', 'c']);
+    });
+
+    it('ignores specified keys that do not exist in the object', () => {
+      const obj = { a: 1, c: 3 };
+      const order = ['a', 'b', 'c', 'd'] as (keyof typeof obj)[];
+      const result = orderKeys(obj, order);
+      expect(Object.keys(result)).toEqual(['a', 'c']);
+    });
+
+    it('returns an empty object when input is empty', () => {
+      const obj = {};
+      const order = ['a', 'b', 'c'] as (keyof typeof obj)[];
+      const result = orderKeys(obj, order);
+      expect(result).toEqual({});
+    });
+
+    it('returns the original object when order is empty', () => {
+      const obj = { c: 3, a: 1, b: 2 };
+      const order: (keyof typeof obj)[] = [];
+      const result = orderKeys(obj, order);
+      expect(result).toEqual(obj);
+    });
+
+    it('preserves nested object structures', () => {
+      const obj = { c: { x: 1 }, a: [1, 2], b: 2 };
+      const order: (keyof typeof obj)[] = ['a', 'b', 'c'];
+      const result = orderKeys(obj, order);
+      expect(result).toEqual({ a: [1, 2], b: 2, c: { x: 1 } });
+    });
+
+    it('handles objects with symbol keys', () => {
+      const sym1 = Symbol('sym1');
+      const sym2 = Symbol('sym2');
+      const obj = { [sym1]: 1, b: 2, [sym2]: 3, a: 4 };
+      const order: (keyof typeof obj)[] = ['a', 'b'];
+      const result = orderKeys(obj, order);
+      expect(Object.getOwnPropertySymbols(result)).toEqual([sym1, sym2]);
+      expect(Object.keys(result)).toEqual(['a', 'b']);
+    });
+
+    it('maintains the correct types for keys and values', () => {
+      const obj = { a: 'string', b: 42, c: true, d: null, e: undefined };
+      const order: (keyof typeof obj)[] = ['b', 'a', 'c', 'd', 'e'];
+      const result = orderKeys(obj, order);
+      expect(result).toEqual({ b: 42, a: 'string', c: true, d: null, e: undefined });
+    });
+  });
+
+  describe('extractFirstJsonObject', () => {
+    it('should extract the first JSON object from a string', () => {
+      const input = '{"key1": "value1"} {"key2": "value2"}';
+      const expected = { key1: 'value1' };
+      expect(extractFirstJsonObject(input)).toEqual(expected);
+    });
+
+    it('should throw an error when no JSON objects are found', () => {
+      const input = 'no json here';
+      expect(() => extractFirstJsonObject(input)).toThrow(
+        'Expected a JSON object, but got "no json here"',
+      );
+    });
+  });
+
+  describe('summarizeEvaluateResultForLogging', () => {
+    // Create test EvaluateResult with reasonable data size
+    const createTestEvaluateResult = (): EvaluateResult =>
+      createEvaluateResult({
+        id: 'test-eval-result',
+        success: false,
+        score: 0.5,
+        error: 'Test error',
+        failureReason: ResultFailureReason.ERROR,
+        latencyMs: 100,
+        promptId: 'test-prompt-id',
+        provider: { id: 'test-provider', label: 'Test Provider' },
+        response: {
+          output: 'This is a test output that is reasonably sized for testing purposes.',
+          error: undefined,
+          cached: false,
+          cost: 0.01,
+          tokenUsage: { total: 1000, prompt: 500, completion: 500 },
+          metadata: {
+            model: 'gpt-4',
+            timestamp: '2024-01-01T00:00:00Z',
+            additionalData: 'Some additional metadata for testing',
+          },
+        },
+        testCase: createAtomicTestCase({
+          description: 'Test case with regular data',
+          vars: { input: 'test input', context: 'test context' },
+        }),
+        prompt: createPrompt('Test prompt', {
+          display: 'Test Prompt Display',
+          label: 'Test Prompt Label',
+        }),
+        vars: {
+          userInput: 'test user input',
+          systemPrompt: 'test system prompt',
+        },
+      });
+
+    it('should throw TypeError for null or undefined input', () => {
+      expect(() => summarizeEvaluateResultForLogging(null as any)).toThrow(TypeError);
+      expect(() => summarizeEvaluateResultForLogging(undefined as any)).toThrow(TypeError);
+    });
+
+    it('should create a safe summary of evaluation results', () => {
+      const testResult = createTestEvaluateResult();
+      const summary = summarizeEvaluateResultForLogging(testResult);
+
+      expect(summary.id).toBe('test-eval-result');
+      expect(summary.testIdx).toBe(0);
+      expect(summary.promptIdx).toBe(0);
+      expect(summary.success).toBe(false);
+      expect(summary.score).toBe(0.5);
+      expect(summary.error).toBe('Test error');
+      expect(summary.failureReason).toBe(ResultFailureReason.ERROR);
+
+      expect(summary.provider?.id).toBe('test-provider');
+      expect(summary.provider?.label).toBe('Test Provider');
+
+      expect(summary.response?.output).toBeDefined();
+      expect(summary.response?.cached).toBe(false);
+      expect(summary.response?.cost).toBe(0.01);
+      expect(summary.response?.tokenUsage).toEqual({
+        total: 1000,
+        prompt: 500,
+        completion: 500,
+      });
+
+      expect(summary.response?.metadata?.keys).toContain('model');
+      expect(summary.response?.metadata?.keys).toContain('timestamp');
+      expect(summary.response?.metadata?.keys).toContain('additionalData');
+      expect(summary.response?.metadata?.keyCount).toBe(3);
+
+      expect(summary.testCase?.description).toBe('Test case with regular data');
+      expect(summary.testCase?.vars).toEqual(['input', 'context']);
+    });
+
+    it('should handle custom options', () => {
+      const testResult = createTestEvaluateResult();
+      const summary = summarizeEvaluateResultForLogging(testResult, 20, false);
+
+      expect(summary.response?.output?.length).toBeLessThanOrEqual(35); // 20 + '...[truncated]'
+      expect(summary.response?.metadata).toBeUndefined();
+    });
+
+    it('should handle EvaluateResult with minimal data', () => {
+      const minimalResult: EvaluateResult = createEvaluateResult({
+        id: 'minimal-result',
+        testIdx: 1,
+        promptIdx: 2,
+        score: 1.0,
+        latencyMs: 50,
+        promptId: 'minimal-prompt-id',
+        provider: { id: 'minimal-provider' },
+        prompt: createPrompt('test prompt', { label: 'Test Prompt' }),
+      });
+
+      const summary = summarizeEvaluateResultForLogging(minimalResult);
+
+      expect(summary.id).toBe('minimal-result');
+      expect(summary.success).toBe(true);
+      expect(summary.score).toBe(1.0);
+      expect(summary.provider?.id).toBe('minimal-provider');
+    });
+
+    it('should handle long output strings by truncating them', () => {
+      const longOutput = 'A'.repeat(1000);
+      const resultWithLongOutput: EvaluateResult = {
+        ...createTestEvaluateResult(),
+        response: {
+          output: longOutput,
+          cached: false,
+        },
+      };
+
+      const summary = summarizeEvaluateResultForLogging(resultWithLongOutput, 100);
+
+      expect(summary.response?.output?.length).toBeLessThanOrEqual(115); // 100 + '...[truncated]'
+      expect(summary.response?.output).toContain('...[truncated]');
+    });
+
+    it('should be safely stringifiable', () => {
+      const testResult = createTestEvaluateResult();
+      const summary = summarizeEvaluateResultForLogging(testResult);
+
+      expect(() => {
+        const result = safeJsonStringify(summary);
+        expect(result).toBeDefined();
+        expect(typeof result).toBe('string');
+      }).not.toThrow();
+    });
+
+    it('should handle non-string output values', () => {
+      const resultWithObjectOutput: EvaluateResult = {
+        ...createTestEvaluateResult(),
+        response: {
+          output: { complexObject: 'value', nested: { data: 'test' } },
+          cached: false,
+        },
+      };
+
+      const summary = summarizeEvaluateResultForLogging(resultWithObjectOutput);
+
+      expect(summary.response?.output).toBe('[object Object]');
+    });
+
+    describe('Integration test - evaluator error logging scenario', () => {
+      it('should handle the exact scenario from evaluator.ts without throwing', () => {
+        const testEvalResult = createTestEvaluateResult();
+        const mockError = new Error('Database connection failed');
+
+        expect(() => {
+          const resultSummary = summarizeEvaluateResultForLogging(testEvalResult);
+          const logMessage = `Error saving result: ${mockError} ${safeJsonStringify(resultSummary)}`;
+
+          expect(logMessage).toBeDefined();
+          expect(logMessage).toContain('Error saving result: Error: Database connection failed');
+          expect(logMessage).toContain('"id":"test-eval-result"');
+          expect(logMessage).toContain('"success":false');
+          expect(logMessage.length).toBeLessThan(5000);
+        }).not.toThrow();
+      });
+
+      it('should perform efficiently', () => {
+        const testResult = createTestEvaluateResult();
+
+        const start = Date.now();
+        const summary = summarizeEvaluateResultForLogging(testResult);
+        const stringified = safeJsonStringify(summary);
+        const duration = Date.now() - start;
+
+        expect(duration).toBeLessThan(100);
+        expect(stringified).toBeDefined();
+      });
+    });
+  });
+});

test/util/jsonExport.test.ts

@@ -0,0 +1,408 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import * as blobs from '../../src/blobs';
+import { getTraceStore } from '../../src/tracing/store';
+import { writeOutput } from '../../src/util/index';
+import { createTempDir, mockProcessEnv, removeTempDir } from './utils';
+
+// Mock dependencies
+vi.mock('../../src/database', () => ({
+  getDb: vi.fn().mockReturnValue({
+    select: vi.fn(),
+    insert: vi.fn(),
+    transaction: vi.fn(),
+  }),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    info: vi.fn(),
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+  },
+}));
+
+describe('JSON export with improved error handling', () => {
+  let tempDir: string;
+  let tempFilePath: string;
+  let mockEval: any;
+
+  beforeEach(() => {
+    tempDir = createTempDir('promptfoo-json-test-');
+    tempFilePath = path.join(tempDir, 'test-export.json');
+
+    mockEval = {
+      id: 'test-eval-id',
+      createdAt: '2025-01-01T00:00:00.000Z',
+      author: 'test-author',
+      config: { testConfig: true },
+      prompts: [
+        { raw: 'Test prompt 1', label: 'prompt1' },
+        { raw: 'Test prompt 2', label: 'prompt2' },
+      ],
+      toEvaluateSummary: vi.fn(),
+      getResultsCount: vi.fn(),
+    };
+  });
+
+  afterEach(() => {
+    removeTempDir(tempDir);
+    vi.clearAllMocks();
+  });
+
+  describe('normal JSON export', () => {
+    beforeEach(() => {
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: [
+          { testIdx: 0, promptIdx: 0, success: true, score: 1.0 },
+          { testIdx: 1, promptIdx: 0, success: true, score: 0.9 },
+        ],
+        stats: {
+          successes: 2,
+          failures: 0,
+          tokenUsage: { total: 100, prompt: 50, completion: 50 },
+        },
+      });
+    });
+
+    it('should export JSON successfully', async () => {
+      await writeOutput(tempFilePath, mockEval, 'https://share.url');
+
+      expect(fs.existsSync(tempFilePath)).toBe(true);
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      // Verify structure matches expected OutputFile format
+      expect(parsed).toHaveProperty('evalId', 'test-eval-id');
+      expect(parsed).toHaveProperty('results');
+      expect(parsed.results).toHaveProperty('version', 3);
+      expect(parsed.results.results).toHaveLength(2);
+      expect(parsed).toHaveProperty('config', { testConfig: true });
+      expect(parsed).toHaveProperty('shareableUrl', 'https://share.url');
+      expect(parsed).toHaveProperty('metadata');
+    });
+
+    it('should export persisted vars and runtime options for round-trip imports', async () => {
+      mockEval.vars = ['topic', 'tone'];
+      mockEval.runtimeOptions = { cache: false, maxConcurrency: 2 };
+
+      await writeOutput(tempFilePath, mockEval, null);
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed).toHaveProperty('vars', ['topic', 'tone']);
+      expect(parsed).toHaveProperty('runtimeOptions', { cache: false, maxConcurrency: 2 });
+    });
+
+    it('should embed referenced blob bytes only when media export is enabled', async () => {
+      const hash = 'a'.repeat(64);
+      const data = Buffer.from('portable image');
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: [{ response: { output: `promptfoo://blob/${hash}` } }],
+        stats: { successes: 1, failures: 0 },
+      });
+      vi.spyOn(blobs, 'getBlobByHash').mockResolvedValue({
+        data,
+        metadata: {
+          mimeType: 'image/png',
+          sizeBytes: data.length,
+          createdAt: '2025-01-01T00:00:00.000Z',
+          provider: 'filesystem',
+          key: hash,
+        },
+      });
+
+      await writeOutput(tempFilePath, mockEval, null);
+      expect(JSON.parse(fs.readFileSync(tempFilePath, 'utf8'))).not.toHaveProperty('blobAssets');
+
+      await writeOutput(tempFilePath, mockEval, null, { includeMedia: true });
+      const parsed = JSON.parse(fs.readFileSync(tempFilePath, 'utf8'));
+      expect(parsed.blobAssets).toEqual([
+        {
+          hash,
+          mimeType: 'image/png',
+          sizeBytes: data.length,
+          data: data.toString('base64'),
+        },
+      ]);
+    });
+
+    it('should embed blob bytes referenced only by exported traces', async () => {
+      const hash = 'b'.repeat(64);
+      const data = Buffer.from('portable trace image');
+      const traceSpy = vi.spyOn(getTraceStore(), 'getTracesByEvaluation').mockResolvedValue([
+        {
+          traceId: 'trace-media-export',
+          evaluationId: mockEval.id,
+          testCaseId: 'trace-media-case',
+          metadata: { attachment: `promptfoo://blob/${hash}` },
+          spans: [],
+        },
+      ]);
+      vi.spyOn(blobs, 'getBlobByHash').mockResolvedValue({
+        data,
+        metadata: {
+          mimeType: 'image/png',
+          sizeBytes: data.length,
+          createdAt: '2025-01-01T00:00:00.000Z',
+          provider: 'filesystem',
+          key: hash,
+        },
+      });
+
+      try {
+        await writeOutput(tempFilePath, mockEval, null, { includeMedia: true });
+
+        const parsed = JSON.parse(fs.readFileSync(tempFilePath, 'utf8'));
+        expect(parsed.traces[0].metadata.attachment).toBe(`promptfoo://blob/${hash}`);
+        expect(parsed.blobAssets).toEqual([
+          {
+            hash,
+            mimeType: 'image/png',
+            sizeBytes: data.length,
+            data: data.toString('base64'),
+          },
+        ]);
+      } finally {
+        traceSpy.mockRestore();
+      }
+    });
+
+    it('should not embed response blob bytes when response output stripping is enabled', async () => {
+      const restoreEnv = mockProcessEnv({ PROMPTFOO_STRIP_RESPONSE_OUTPUT: 'true' });
+      const hash = 'c'.repeat(64);
+      const getBlobSpy = vi.spyOn(blobs, 'getBlobByHash');
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: [
+          {
+            response: {
+              output: '[output stripped]',
+              metadata: { blobUris: [`promptfoo://blob/${hash}`] },
+            },
+          },
+        ],
+        stats: { successes: 1, failures: 0 },
+      });
+
+      try {
+        await writeOutput(tempFilePath, mockEval, null, { includeMedia: true });
+
+        const parsed = JSON.parse(fs.readFileSync(tempFilePath, 'utf8'));
+        expect(parsed.results.results[0].response.metadata.blobUris).toEqual([
+          `promptfoo://blob/${hash}`,
+        ]);
+        expect(parsed).not.toHaveProperty('blobAssets');
+        expect(getBlobSpy).not.toHaveBeenCalled();
+      } finally {
+        getBlobSpy.mockRestore();
+        restoreEnv();
+      }
+    });
+
+    it('should handle null shareableUrl', async () => {
+      await writeOutput(tempFilePath, mockEval, null);
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.shareableUrl).toBeNull();
+    });
+
+    it('should maintain proper JSON formatting', async () => {
+      await writeOutput(tempFilePath, mockEval, 'https://test.url');
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+
+      // Verify proper 2-space indentation
+      expect(content).toContain('{\n  "evalId":');
+      expect(content).toContain('  "results": {');
+      expect(content).toContain('    "version":');
+
+      // Should be valid JSON
+      expect(() => JSON.parse(content)).not.toThrow();
+    });
+
+    it('should include all required metadata fields', async () => {
+      await writeOutput(tempFilePath, mockEval, 'https://test.url');
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.metadata).toHaveProperty('promptfooVersion');
+      expect(parsed.metadata).toHaveProperty('nodeVersion');
+      expect(parsed.metadata).toHaveProperty('platform');
+      expect(parsed.metadata).toHaveProperty('exportedAt');
+      expect(parsed.metadata).toHaveProperty('author', 'test-author');
+    });
+  });
+
+  describe('memory limit error handling', () => {
+    beforeEach(() => {
+      mockEval.getResultsCount.mockResolvedValue(50000);
+      mockEval.toEvaluateSummary.mockImplementation(() => {
+        throw new RangeError('Invalid string length');
+      });
+    });
+
+    it('should handle RangeError gracefully with helpful message', async () => {
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+        'Dataset too large for JSON export',
+      );
+
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+        'Consider using JSONL format instead',
+      );
+    });
+
+    it('should include result count in error message', async () => {
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow('50000 results');
+    });
+
+    it('should not create output file when memory error occurs', async () => {
+      try {
+        await writeOutput(tempFilePath, mockEval, null);
+      } catch (_error) {
+        // Expected to throw
+      }
+
+      expect(fs.existsSync(tempFilePath)).toBe(false);
+    });
+  });
+
+  describe('other error handling', () => {
+    it('should propagate non-RangeError exceptions', async () => {
+      const testError = new Error('Database connection failed');
+      mockEval.toEvaluateSummary.mockRejectedValue(testError);
+
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+        'Database connection failed',
+      );
+    });
+
+    it('should handle file system errors', async () => {
+      // Use a path that will definitely fail - writing to a file that already exists as a directory
+      const dirAsFile = path.join(tempDir, 'directory-as-file');
+      fs.mkdirSync(dirAsFile);
+      const invalidPath = dirAsFile; // Try to write to a directory path as if it were a file
+
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        results: [],
+        stats: { successes: 0, failures: 0 },
+      });
+
+      await expect(writeOutput(invalidPath, mockEval, null)).rejects.toThrow();
+    });
+  });
+
+  describe('backward compatibility', () => {
+    it('should maintain exact same JSON structure as original implementation', async () => {
+      const expectedSummary = {
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: [
+          { raw: 'Test prompt 1', label: 'prompt1' },
+          { raw: 'Test prompt 2', label: 'prompt2' },
+        ],
+        results: [
+          {
+            testIdx: 0,
+            promptIdx: 0,
+            success: true,
+            score: 1.0,
+            vars: { input: 'test' },
+            output: 'response',
+          },
+        ],
+        stats: {
+          successes: 1,
+          failures: 0,
+          errors: 0,
+          tokenUsage: { total: 100, prompt: 50, completion: 50 },
+        },
+      };
+
+      mockEval.toEvaluateSummary.mockResolvedValue(expectedSummary);
+
+      await writeOutput(tempFilePath, mockEval, 'https://share.url');
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      // Verify exact structure
+      expect(Object.keys(parsed).sort()).toEqual(
+        ['evalId', 'results', 'config', 'shareableUrl', 'metadata'].sort(),
+      );
+
+      expect(parsed.results).toEqual(expectedSummary);
+    });
+
+    it('should handle both EvaluateSummaryV2 and V3 formats', async () => {
+      const v2Summary = {
+        version: 2,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        results: [{ testIdx: 0, success: true }],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: { successes: 1, failures: 0 },
+      };
+
+      mockEval.toEvaluateSummary.mockResolvedValue(v2Summary);
+
+      await writeOutput(tempFilePath, mockEval, null);
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.results.version).toBe(2);
+      expect(parsed.results).toHaveProperty('table');
+      expect(parsed.results.results).toEqual([{ testIdx: 0, success: true }]);
+    });
+  });
+
+  describe('performance considerations', () => {
+    it('should complete export in reasonable time for moderate datasets', async () => {
+      // Create a moderate-sized dataset
+      const moderateResults = Array.from({ length: 1000 }, (_, i) => ({
+        testIdx: i,
+        promptIdx: 0,
+        success: true,
+        score: Math.random(),
+        vars: { input: `test input ${i}` },
+        output: `test output ${i}`,
+      }));
+
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: moderateResults,
+        stats: { successes: 1000, failures: 0 },
+      });
+
+      const startTime = Date.now();
+      await writeOutput(tempFilePath, mockEval, null);
+      const duration = Date.now() - startTime;
+
+      // Should complete within reasonable time (5 seconds for 1000 results)
+      expect(duration).toBeLessThan(5000);
+
+      // Verify file was created and has content
+      expect(fs.existsSync(tempFilePath)).toBe(true);
+      const stats = fs.statSync(tempFilePath);
+      expect(stats.size).toBeGreaterThan(1000); // Should have substantial content
+    }, 10000); // 10 second timeout
+  });
+});

test/util/jsonlOutput.test.ts

@@ -0,0 +1,253 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { writeOutput } from '../../src/util/index';
+import { createTempDir, removeTempDir } from './utils';
+
+// Mock dependencies
+vi.mock('../../src/database', () => ({
+  getDb: vi.fn().mockReturnValue({
+    select: vi.fn(),
+    insert: vi.fn(),
+    transaction: vi.fn(),
+  }),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    info: vi.fn(),
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+  },
+}));
+
+describe('JSONL output with proper line endings', () => {
+  let tempDir: string;
+  let tempFilePath: string;
+  let mockEval: any;
+
+  beforeEach(() => {
+    tempDir = createTempDir('promptfoo-jsonl-test-');
+    tempFilePath = path.join(tempDir, 'test-export.jsonl');
+
+    mockEval = {
+      id: 'test-eval-id',
+      createdAt: '2025-01-01T00:00:00.000Z',
+      author: 'test-author',
+      config: { testConfig: true },
+      prompts: [
+        { raw: 'Test prompt 1', label: 'prompt1' },
+        { raw: 'Test prompt 2', label: 'prompt2' },
+      ],
+      fetchResultsBatched: vi.fn(),
+    };
+  });
+
+  afterEach(() => {
+    removeTempDir(tempDir);
+    vi.clearAllMocks();
+  });
+
+  it('should produce valid JSONL with proper line endings between batches', async () => {
+    // Mock fetchResultsBatched to return multiple batches
+    const batch1 = [
+      { testIdx: 0, success: true, score: 1.0, output: 'result 1' },
+      { testIdx: 1, success: true, score: 0.9, output: 'result 2' },
+    ];
+    const batch2 = [
+      { testIdx: 2, success: true, score: 0.8, output: 'result 3' },
+      { testIdx: 3, success: false, score: 0.0, output: 'result 4' },
+    ];
+
+    // Create async iterator for batches
+    const batchIterator = [batch1, batch2];
+
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      for (const batch of batchIterator) {
+        yield batch;
+      }
+    });
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    expect(fs.existsSync(tempFilePath)).toBe(true);
+    const content = fs.readFileSync(tempFilePath, 'utf8');
+
+    // Split by lines and filter out empty lines
+    const lines = content.split(/\r?\n/).filter((line) => line.trim() !== '');
+
+    // Should have exactly 4 JSON objects (2 per batch)
+    expect(lines).toHaveLength(4);
+
+    // Each line should be valid JSON
+    lines.forEach((line, index) => {
+      expect(() => JSON.parse(line)).not.toThrow();
+      const parsed = JSON.parse(line);
+      expect(parsed).toHaveProperty('testIdx', index);
+      expect(parsed).toHaveProperty('output', `result ${index + 1}`);
+    });
+
+    // File should end with a newline
+    expect(content.endsWith('\n')).toBe(true);
+  });
+
+  it('should handle single batch correctly', async () => {
+    const singleBatch = [{ testIdx: 0, success: true, score: 1.0, output: 'single result' }];
+
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield singleBatch;
+    });
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    const content = fs.readFileSync(tempFilePath, 'utf8');
+    const lines = content.split(/\r?\n/).filter((line) => line.trim() !== '');
+
+    expect(lines).toHaveLength(1);
+    expect(() => JSON.parse(lines[0])).not.toThrow();
+    expect(content.endsWith('\n')).toBe(true);
+  });
+
+  it('should handle empty batches gracefully', async () => {
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield [];
+      yield [{ testIdx: 0, success: true, score: 1.0, output: 'after empty' }];
+      yield [];
+    });
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    const content = fs.readFileSync(tempFilePath, 'utf8');
+    const lines = content.split(/\r?\n/).filter((line) => line.trim() !== '');
+
+    expect(lines).toHaveLength(1);
+    expect(() => JSON.parse(lines[0])).not.toThrow();
+    const parsed = JSON.parse(lines[0]);
+    expect(parsed.output).toBe('after empty');
+  });
+
+  it('should use OS-specific line endings', async () => {
+    const batch = [{ testIdx: 0, success: true, score: 1.0, output: 'test result' }];
+
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield batch;
+    });
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    const content = fs.readFileSync(tempFilePath, 'utf8');
+
+    // Content should end with the OS-specific line ending
+    expect(content.endsWith(os.EOL)).toBe(true);
+
+    // For multiple results, should use OS line endings between them
+    const multiResultBatch = [
+      { testIdx: 0, success: true, score: 1.0, output: 'result 1' },
+      { testIdx: 1, success: true, score: 0.9, output: 'result 2' },
+    ];
+
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield multiResultBatch;
+    });
+
+    // Clear the file
+    fs.unlinkSync(tempFilePath);
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    const multiContent = fs.readFileSync(tempFilePath, 'utf8');
+    const expectedContent =
+      multiResultBatch.map((result) => JSON.stringify(result)).join(os.EOL) + os.EOL;
+
+    expect(multiContent).toBe(expectedContent);
+  });
+
+  it('should preserve the existing artifact when a rewrite fails', async () => {
+    fs.writeFileSync(tempFilePath, '{"status":"existing"}\n');
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield [{ testIdx: 0, success: true, score: 1.0, output: 'replacement' }];
+      throw new Error('simulated rewrite failure');
+    });
+
+    await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+      'simulated rewrite failure',
+    );
+
+    expect(fs.readFileSync(tempFilePath, 'utf8')).toBe('{"status":"existing"}\n');
+    expect(fs.readdirSync(tempDir)).toEqual(['test-export.jsonl']);
+  });
+
+  it('should preserve existing artifact permissions during a rewrite', async () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+
+    fs.writeFileSync(tempFilePath, '{"status":"existing"}\n', { mode: 0o644 });
+    fs.chmodSync(tempFilePath, 0o644);
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield [{ testIdx: 0, success: true, score: 1.0, output: 'replacement' }];
+    });
+
+    const originalUmask = process.umask(0o077);
+    try {
+      await writeOutput(tempFilePath, mockEval, null);
+    } finally {
+      process.umask(originalUmask);
+    }
+
+    expect(fs.statSync(tempFilePath).mode & 0o777).toBe(0o644);
+  });
+
+  it('should preserve an existing artifact symlink during a rewrite', async () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+
+    const targetPath = path.join(tempDir, 'target.jsonl');
+    fs.writeFileSync(targetPath, '{"status":"existing"}\n');
+    fs.symlinkSync(targetPath, tempFilePath);
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield [{ testIdx: 0, success: true, score: 1.0, output: 'replacement' }];
+    });
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    expect(fs.lstatSync(tempFilePath).isSymbolicLink()).toBe(true);
+    expect(fs.readFileSync(targetPath, 'utf8')).toContain('"output":"replacement"');
+  });
+
+  it('should preserve a dangling artifact symlink and create its target during a rewrite', async () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+
+    const targetPath = path.join(tempDir, 'missing-target.jsonl');
+    fs.symlinkSync(path.basename(targetPath), tempFilePath);
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield [{ testIdx: 0, success: true, score: 1.0, output: 'replacement' }];
+    });
+
+    await writeOutput(tempFilePath, mockEval, null);
+
+    expect(fs.lstatSync(tempFilePath).isSymbolicLink()).toBe(true);
+    expect(fs.readFileSync(targetPath, 'utf8')).toContain('"output":"replacement"');
+  });
+
+  it('should rewrite an artifact with a long valid basename', async () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+
+    const longFilePath = path.join(tempDir, `${'a'.repeat(240)}.jsonl`);
+    mockEval.fetchResultsBatched = vi.fn().mockImplementation(async function* () {
+      yield [{ testIdx: 0, success: true, score: 1.0, output: 'replacement' }];
+    });
+
+    await writeOutput(longFilePath, mockEval, null);
+
+    expect(fs.readFileSync(longFilePath, 'utf8')).toContain('"output":"replacement"');
+  });
+});

test/util/libsqlBindingErrors.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest';
+import {
+  formatLibsqlBindingErrorMessage,
+  getLibsqlBindingTarget,
+} from '../../src/util/libsqlBindingErrors';
+
+function makeBindingError(target: string): NodeJS.ErrnoException {
+  const error: NodeJS.ErrnoException = new Error(
+    `Cannot find module '@libsql/${target}'\nRequire stack:\n- /app/node_modules/libsql/index.js`,
+  );
+  error.code = 'MODULE_NOT_FOUND';
+  return error;
+}
+
+describe('libsql binding errors', () => {
+  it('extracts the missing libsql target from ESM import failures', () => {
+    const error: NodeJS.ErrnoException = new Error(
+      "Cannot find package '@libsql/linux-x64-gnu' imported from /app/node_modules/@libsql/client/lib-esm/node.js",
+    );
+    error.code = 'ERR_MODULE_NOT_FOUND';
+
+    expect(getLibsqlBindingTarget(error)).toBe('linux-x64-gnu');
+  });
+
+  it('extracts the missing libsql target from the require stack', () => {
+    expect(getLibsqlBindingTarget(makeBindingError('darwin-arm64'))).toBe('darwin-arm64');
+    expect(getLibsqlBindingTarget(makeBindingError('linux-x64-musl'))).toBe('linux-x64-musl');
+  });
+
+  it('formats actionable repair instructions with the detected target and platform', () => {
+    const message = formatLibsqlBindingErrorMessage(makeBindingError('linux-x64-gnu'));
+
+    expect(message).toContain('Required package: @libsql/linux-x64-gnu');
+    expect(message).toContain(`${process.platform}-${process.arch}`);
+    expect(message).toContain('libsql-binding-not-found');
+  });
+
+  it('ignores MODULE_NOT_FOUND errors that are not from libsql', () => {
+    const error: NodeJS.ErrnoException = new Error("Cannot find module 'some-other-pkg'");
+    error.code = 'MODULE_NOT_FOUND';
+
+    expect(getLibsqlBindingTarget(error)).toBeUndefined();
+    expect(formatLibsqlBindingErrorMessage(error)).toBeUndefined();
+  });
+
+  it('ignores missing libsql wrapper packages that are not platform bindings', () => {
+    for (const packageName of ['client', 'client-wasm', 'core', 'hrana-client']) {
+      const error: NodeJS.ErrnoException = new Error(
+        `Cannot find module '@libsql/${packageName}'\nRequire stack:\n- /app/index.js`,
+      );
+      error.code = 'MODULE_NOT_FOUND';
+
+      expect(getLibsqlBindingTarget(error)).toBeUndefined();
+      expect(formatLibsqlBindingErrorMessage(error)).toBeUndefined();
+    }
+  });
+
+  it('ignores libsql-mentioning errors that are not MODULE_NOT_FOUND', () => {
+    const error: NodeJS.ErrnoException = new Error("Cannot find module '@libsql/darwin-arm64'");
+    // No `code` set — looks like the right message but wrong shape.
+
+    expect(getLibsqlBindingTarget(error)).toBeUndefined();
+  });
+
+  it('ignores non-Error values', () => {
+    expect(getLibsqlBindingTarget('boom')).toBeUndefined();
+    expect(getLibsqlBindingTarget(undefined)).toBeUndefined();
+    expect(formatLibsqlBindingErrorMessage(null)).toBeUndefined();
+  });
+});

test/util/logFiles.test.ts

@@ -0,0 +1,181 @@
+import fs from 'fs';
+import path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getLogFiles } from '../../src/util/logFiles';
+
+vi.mock('fs');
+
+// Helper to create mock Dirent objects
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function createDirent(name: string, isFile: boolean = true): any {
+  return {
+    name,
+    isFile: () => isFile,
+    isDirectory: () => !isFile,
+    isBlockDevice: () => false,
+    isCharacterDevice: () => false,
+    isSymbolicLink: () => false,
+    isFIFO: () => false,
+    isSocket: () => false,
+    parentPath: '/logs',
+    path: '/logs',
+  };
+}
+
+describe('logFiles utilities', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe('getLogFiles', () => {
+    it('should return empty array if directory does not exist', () => {
+      const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
+      error.code = 'ENOENT';
+      vi.mocked(fs.readdirSync).mockImplementation(() => {
+        throw error;
+      });
+
+      const result = getLogFiles('/nonexistent/path');
+
+      expect(result).toEqual([]);
+    });
+
+    it('should return empty array if directory has no log files', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([
+        createDirent('readme.txt'),
+        createDirent('config.json'),
+      ]);
+
+      const result = getLogFiles('/logs');
+
+      expect(result).toEqual([]);
+    });
+
+    it('should return log files sorted by modification time (newest first)', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([
+        createDirent('promptfoo-debug-2024-01-01.log'),
+        createDirent('promptfoo-error-2024-01-02.log'),
+        createDirent('promptfoo-debug-2024-01-03.log'),
+      ]);
+
+      const dates = {
+        'promptfoo-debug-2024-01-01.log': new Date('2024-01-01'),
+        'promptfoo-error-2024-01-02.log': new Date('2024-01-02'),
+        'promptfoo-debug-2024-01-03.log': new Date('2024-01-03'),
+      };
+
+      vi.mocked(fs.statSync).mockImplementation((filePath: unknown) => {
+        const fileName = path.basename(filePath as string);
+        return { mtime: dates[fileName as keyof typeof dates] } as fs.Stats;
+      });
+
+      const result = getLogFiles('/logs');
+
+      expect(result).toHaveLength(3);
+      expect(result[0].name).toBe('promptfoo-debug-2024-01-03.log');
+      expect(result[1].name).toBe('promptfoo-error-2024-01-02.log');
+      expect(result[2].name).toBe('promptfoo-debug-2024-01-01.log');
+    });
+
+    it('should filter files by promptfoo prefix and .log suffix', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([
+        createDirent('promptfoo-debug.log'),
+        createDirent('other-debug.log'),
+        createDirent('promptfoo-error.txt'),
+        createDirent('promptfoo.log'),
+        createDirent('readme.md'),
+      ]);
+
+      vi.mocked(fs.statSync).mockReturnValue({ mtime: new Date() } as fs.Stats);
+
+      const result = getLogFiles('/logs');
+
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toBe('promptfoo-debug.log');
+    });
+
+    it('should filter out directories', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([
+        createDirent('promptfoo-debug.log', true),
+        createDirent('promptfoo-subdir.log', false), // This is a directory
+      ]);
+
+      vi.mocked(fs.statSync).mockReturnValue({ mtime: new Date() } as fs.Stats);
+
+      const result = getLogFiles('/logs');
+
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toBe('promptfoo-debug.log');
+    });
+
+    it('should include correct path in returned objects', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([createDirent('promptfoo-test.log')]);
+      vi.mocked(fs.statSync).mockReturnValue({ mtime: new Date() } as fs.Stats);
+
+      const result = getLogFiles('/my/log/dir');
+
+      expect(result[0].path).toBe(path.join('/my/log/dir', 'promptfoo-test.log'));
+    });
+
+    it('should throw error if directory cannot be read (non-ENOENT)', () => {
+      const error = new Error('Permission denied') as NodeJS.ErrnoException;
+      error.code = 'EACCES';
+      vi.mocked(fs.readdirSync).mockImplementation(() => {
+        throw error;
+      });
+
+      expect(() => getLogFiles('/protected/logs')).toThrow('Permission denied');
+    });
+
+    it('should skip files that are deleted between readdir and stat (race condition)', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([
+        createDirent('promptfoo-exists.log'),
+        createDirent('promptfoo-deleted.log'),
+        createDirent('promptfoo-also-exists.log'),
+      ]);
+
+      vi.mocked(fs.statSync).mockImplementation((filePath: unknown) => {
+        if ((filePath as string).includes('deleted')) {
+          const error = new Error('ENOENT: no such file or directory') as NodeJS.ErrnoException;
+          error.code = 'ENOENT';
+          throw error;
+        }
+        return { mtime: new Date() } as fs.Stats;
+      });
+
+      const result = getLogFiles('/logs');
+
+      expect(result).toHaveLength(2);
+      expect(result.map((f) => f.name)).toContain('promptfoo-exists.log');
+      expect(result.map((f) => f.name)).toContain('promptfoo-also-exists.log');
+      expect(result.map((f) => f.name)).not.toContain('promptfoo-deleted.log');
+    });
+
+    it('should rethrow non-ENOENT errors from statSync', () => {
+      vi.mocked(fs.readdirSync).mockReturnValue([createDirent('promptfoo-test.log')]);
+
+      vi.mocked(fs.statSync).mockImplementation(() => {
+        const error = new Error('Permission denied') as NodeJS.ErrnoException;
+        error.code = 'EACCES';
+        throw error;
+      });
+
+      expect(() => getLogFiles('/logs')).toThrow('Permission denied');
+    });
+
+    it('should return files with correct mtime', () => {
+      const testDate = new Date('2024-06-15T10:30:00Z');
+      vi.mocked(fs.readdirSync).mockReturnValue([createDirent('promptfoo-test.log')]);
+      vi.mocked(fs.statSync).mockReturnValue({ mtime: testDate } as fs.Stats);
+
+      const result = getLogFiles('/logs');
+
+      expect(result[0].mtime).toEqual(testDate);
+    });
+  });
+});

test/util/logs.test.ts

@@ -0,0 +1,329 @@
+import fsSync from 'fs';
+import fs from 'fs/promises';
+import path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Mock dependencies before importing the module under test
+vi.mock('fs/promises');
+vi.mock('fs');
+vi.mock('../../src/logger', () => ({
+  default: {
+    error: vi.fn(),
+    warn: vi.fn(),
+    info: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+const mockGetConfigDirectoryPath = vi.fn().mockReturnValue('/home/user/.promptfoo');
+vi.mock('../../src/util/config/manage', () => ({
+  getConfigDirectoryPath: () => mockGetConfigDirectoryPath(),
+}));
+
+const mockGetEnvString = vi.fn().mockReturnValue('');
+vi.mock('../../src/envars', () => ({
+  getEnvString: (key: string) => mockGetEnvString(key),
+  getEnvBool: vi.fn().mockReturnValue(false),
+}));
+
+// Import after mocks are set up
+import {
+  findLogFile,
+  formatFileSize,
+  getLogDirectory,
+  getLogFiles,
+  getLogFilesSync,
+} from '../../src/util/logs';
+
+describe('util/logs', () => {
+  const mockFs = vi.mocked(fs);
+  const mockFsSync = vi.mocked(fsSync);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockGetConfigDirectoryPath.mockReturnValue('/home/user/.promptfoo');
+    mockGetEnvString.mockReturnValue('');
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  describe('getLogDirectory', () => {
+    it('returns default directory when PROMPTFOO_LOG_DIR not set', () => {
+      mockGetEnvString.mockReturnValue('');
+
+      const result = getLogDirectory();
+      expect(result).toBe(path.join('/home/user/.promptfoo', 'logs'));
+    });
+
+    it('respects PROMPTFOO_LOG_DIR environment variable', () => {
+      mockGetEnvString.mockReturnValue('/custom/log/dir');
+
+      const result = getLogDirectory();
+      expect(result).toBe(path.resolve('/custom/log/dir'));
+    });
+  });
+
+  describe('getLogFiles (async)', () => {
+    beforeEach(() => {
+      mockGetEnvString.mockReturnValue('');
+    });
+
+    it('returns empty array when directory does not exist', async () => {
+      mockFs.access.mockRejectedValue(new Error('ENOENT'));
+
+      const result = await getLogFiles();
+      expect(result).toEqual([]);
+    });
+
+    it('returns empty array when no log files found', async () => {
+      mockFs.access.mockResolvedValue(undefined);
+      mockFs.readdir.mockResolvedValue([]);
+
+      const result = await getLogFiles();
+      expect(result).toEqual([]);
+    });
+
+    it('filters non-promptfoo files', async () => {
+      mockFs.access.mockResolvedValue(undefined);
+      mockFs.readdir.mockResolvedValue([
+        'other-file.log',
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'readme.txt',
+      ] as any);
+      mockFs.stat.mockResolvedValue({
+        mtime: new Date('2024-01-01T10:00:00Z'),
+        size: 1024,
+      } as any);
+
+      const result = await getLogFiles();
+
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toBe('promptfoo-debug-2024-01-01_10-00-00.log');
+    });
+
+    it('filters by type when specified', async () => {
+      mockFs.access.mockResolvedValue(undefined);
+      mockFs.readdir.mockResolvedValue([
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'promptfoo-error-2024-01-01_10-00-00.log',
+      ] as any);
+      mockFs.stat.mockResolvedValue({
+        mtime: new Date('2024-01-01T10:00:00Z'),
+        size: 1024,
+      } as any);
+
+      const debugFiles = await getLogFiles('debug');
+      expect(debugFiles).toHaveLength(1);
+      expect(debugFiles[0].type).toBe('debug');
+
+      const errorFiles = await getLogFiles('error');
+      expect(errorFiles).toHaveLength(1);
+      expect(errorFiles[0].type).toBe('error');
+
+      const allFiles = await getLogFiles('all');
+      expect(allFiles).toHaveLength(2);
+    });
+
+    it('sorts by modification time (newest first)', async () => {
+      mockFs.access.mockResolvedValue(undefined);
+      mockFs.readdir.mockResolvedValue([
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'promptfoo-debug-2024-01-02_10-00-00.log',
+      ] as any);
+
+      const dates = [new Date('2024-01-01T10:00:00Z'), new Date('2024-01-02T10:00:00Z')];
+      let callIndex = 0;
+      mockFs.stat.mockImplementation(
+        async () =>
+          ({
+            mtime: dates[callIndex++ % 2],
+            size: 1024,
+          }) as any,
+      );
+
+      const result = await getLogFiles();
+
+      expect(result).toHaveLength(2);
+      // Newest should be first
+      expect(result[0].mtime.getTime()).toBeGreaterThanOrEqual(result[1].mtime.getTime());
+    });
+
+    it('correctly identifies log types', async () => {
+      mockFs.access.mockResolvedValue(undefined);
+      mockFs.readdir.mockResolvedValue([
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'promptfoo-error-2024-01-01_10-00-00.log',
+      ] as any);
+      mockFs.stat.mockResolvedValue({
+        mtime: new Date('2024-01-01T10:00:00Z'),
+        size: 1024,
+      } as any);
+
+      const result = await getLogFiles('all');
+
+      const debugFile = result.find((f) => f.name.includes('-debug-'));
+      const errorFile = result.find((f) => f.name.includes('-error-'));
+
+      expect(debugFile?.type).toBe('debug');
+      expect(errorFile?.type).toBe('error');
+    });
+  });
+
+  describe('getLogFilesSync', () => {
+    beforeEach(() => {
+      mockGetEnvString.mockReturnValue('');
+    });
+
+    it('returns empty array when directory does not exist', () => {
+      mockFsSync.existsSync.mockReturnValue(false);
+
+      const result = getLogFilesSync();
+      expect(result).toEqual([]);
+    });
+
+    it('returns empty array when no log files found', () => {
+      mockFsSync.existsSync.mockReturnValue(true);
+      mockFsSync.readdirSync.mockReturnValue([]);
+
+      const result = getLogFilesSync();
+      expect(result).toEqual([]);
+    });
+
+    it('filters non-promptfoo files', () => {
+      mockFsSync.existsSync.mockReturnValue(true);
+      mockFsSync.readdirSync.mockReturnValue([
+        'other-file.log',
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'readme.txt',
+      ] as any);
+      mockFsSync.statSync.mockReturnValue({
+        mtime: new Date('2024-01-01T10:00:00Z'),
+        size: 1024,
+      } as any);
+
+      const result = getLogFilesSync();
+
+      expect(result).toHaveLength(1);
+      expect(result[0].name).toBe('promptfoo-debug-2024-01-01_10-00-00.log');
+    });
+
+    it('filters by type when specified', () => {
+      mockFsSync.existsSync.mockReturnValue(true);
+      mockFsSync.readdirSync.mockReturnValue([
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'promptfoo-error-2024-01-01_10-00-00.log',
+      ] as any);
+      mockFsSync.statSync.mockReturnValue({
+        mtime: new Date('2024-01-01T10:00:00Z'),
+        size: 1024,
+      } as any);
+
+      const debugFiles = getLogFilesSync('debug');
+      expect(debugFiles).toHaveLength(1);
+      expect(debugFiles[0].type).toBe('debug');
+
+      const errorFiles = getLogFilesSync('error');
+      expect(errorFiles).toHaveLength(1);
+      expect(errorFiles[0].type).toBe('error');
+
+      const allFiles = getLogFilesSync('all');
+      expect(allFiles).toHaveLength(2);
+    });
+  });
+
+  describe('findLogFile', () => {
+    beforeEach(() => {
+      mockFsSync.existsSync.mockReturnValue(true);
+      mockGetEnvString.mockReturnValue('');
+    });
+
+    it('resolves absolute paths directly', () => {
+      const absolutePath = '/absolute/path/to/logfile.log';
+      mockFsSync.existsSync.mockImplementation((p) => p === absolutePath);
+
+      const result = findLogFile(absolutePath);
+      expect(result).toBe(absolutePath);
+    });
+
+    it('resolves filenames in log directory', () => {
+      const filename = 'promptfoo-debug-2024-01-01_10-00-00.log';
+      const expectedPath = path.join('/home/user/.promptfoo/logs', filename);
+
+      mockFsSync.existsSync.mockImplementation((p) => {
+        // Return false for absolute path check, true for log directory path
+        if (p === filename) {
+          return false;
+        }
+        return p === expectedPath;
+      });
+
+      const result = findLogFile(filename);
+      expect(result).toBe(expectedPath);
+    });
+
+    it('performs fuzzy matching on partial names', () => {
+      const logDir = path.join('/home/user/.promptfoo', 'logs');
+      mockFsSync.readdirSync.mockReturnValue([
+        'promptfoo-debug-2024-01-01_10-00-00.log',
+        'promptfoo-debug-2024-01-02_10-00-00.log',
+      ] as any);
+      mockFsSync.statSync.mockReturnValue({
+        mtime: new Date('2024-01-01T10:00:00Z'),
+        size: 1024,
+      } as any);
+
+      // Make existsSync return false for direct paths to trigger fuzzy matching
+      mockFsSync.existsSync.mockImplementation((p) => {
+        const pStr = String(p);
+        return pStr === logDir;
+      });
+
+      const result = findLogFile('2024-01-01');
+      expect(result).toContain('2024-01-01');
+    });
+
+    it('returns null when file not found', () => {
+      mockFsSync.existsSync.mockReturnValue(false);
+      mockFsSync.readdirSync.mockReturnValue([]);
+
+      const result = findLogFile('nonexistent.log');
+      expect(result).toBeNull();
+    });
+  });
+
+  describe('formatFileSize', () => {
+    it('formats zero bytes', () => {
+      expect(formatFileSize(0)).toBe('0 B');
+    });
+
+    it('formats bytes', () => {
+      expect(formatFileSize(512)).toBe('512 B');
+    });
+
+    it('formats kilobytes', () => {
+      expect(formatFileSize(1024)).toBe('1.0 KB');
+      expect(formatFileSize(1536)).toBe('1.5 KB');
+    });
+
+    it('formats megabytes', () => {
+      expect(formatFileSize(1024 * 1024)).toBe('1.0 MB');
+      expect(formatFileSize(1.5 * 1024 * 1024)).toBe('1.5 MB');
+    });
+
+    it('formats gigabytes', () => {
+      expect(formatFileSize(1024 * 1024 * 1024)).toBe('1.0 GB');
+    });
+
+    it('formats terabytes', () => {
+      expect(formatFileSize(1024 * 1024 * 1024 * 1024)).toBe('1.0 TB');
+    });
+
+    it('handles very large files without overflow', () => {
+      // 10 PB - should cap at TB
+      expect(formatFileSize(10 * 1024 * 1024 * 1024 * 1024 * 1024)).toBe('10240.0 TB');
+    });
+  });
+});

test/util/mockChildProcess.ts

@@ -0,0 +1,133 @@
+/**
+ * Test utility for creating mock ChildProcess objects
+ *
+ * This utility standardizes mock ChildProcess creation across tests,
+ * ensuring all required properties (killed, kill, etc.) are included
+ * and event handlers are properly set up.
+ */
+import type { ChildProcess } from 'child_process';
+
+import { vi } from 'vitest';
+
+export interface MockChildProcessOptions {
+  /** Exit code to return on 'close' event (default: 0) */
+  exitCode?: number | null;
+  /** Error to emit on 'error' event (if set, close won't fire) */
+  error?: Error;
+  /** Data to emit on stdout 'data' event */
+  stdoutData?: string | Buffer;
+  /** Data to emit on stderr 'data' event */
+  stderrData?: string | Buffer;
+  /** Whether the process is killed (default: false) */
+  killed?: boolean;
+  /** Custom event handlers for advanced scenarios */
+  customEventHandlers?: Record<string, (callback: (...args: any[]) => void) => void>;
+}
+
+export interface MockChildProcess {
+  stdout: { on: ReturnType<typeof vi.fn> };
+  stderr: { on: ReturnType<typeof vi.fn> };
+  killed: boolean;
+  kill: ReturnType<typeof vi.fn>;
+  on: ReturnType<typeof vi.fn>;
+}
+
+/**
+ * Creates a mock ChildProcess object with proper event simulation.
+ *
+ * By default, the mock:
+ * - Has `killed: false` and `kill: vi.fn()` for signal handling
+ * - Emits 'close' event with exitCode 0
+ * - Has empty stdout/stderr handlers
+ *
+ * @example Basic usage - successful process
+ * ```typescript
+ * const mock = createMockChildProcess({ exitCode: 0 });
+ * (spawn as Mock).mockReturnValue(mock);
+ * ```
+ *
+ * @example Process with output
+ * ```typescript
+ * const mock = createMockChildProcess({
+ *   exitCode: 0,
+ *   stdoutData: JSON.stringify({ result: 'success' }),
+ * });
+ * ```
+ *
+ * @example Process that errors
+ * ```typescript
+ * const mock = createMockChildProcess({
+ *   error: new Error('spawn failed'),
+ * });
+ * ```
+ *
+ * @example Process with non-zero exit
+ * ```typescript
+ * const mock = createMockChildProcess({
+ *   exitCode: 1,
+ *   stderrData: 'Error: something went wrong',
+ * });
+ * ```
+ */
+export function createMockChildProcess(options: MockChildProcessOptions = {}): MockChildProcess {
+  const {
+    exitCode = 0,
+    error,
+    stdoutData,
+    stderrData,
+    killed = false,
+    customEventHandlers,
+  } = options;
+
+  const mockProcess: MockChildProcess = {
+    stdout: {
+      on: vi.fn().mockImplementation((event: string, callback: (data: Buffer) => void) => {
+        if (event === 'data' && stdoutData) {
+          const data = typeof stdoutData === 'string' ? Buffer.from(stdoutData) : stdoutData;
+          // Use setImmediate to simulate async event emission
+          setImmediate(() => callback(data));
+        }
+      }),
+    },
+    stderr: {
+      on: vi.fn().mockImplementation((event: string, callback: (data: Buffer) => void) => {
+        if (event === 'data' && stderrData) {
+          const data = typeof stderrData === 'string' ? Buffer.from(stderrData) : stderrData;
+          setImmediate(() => callback(data));
+        }
+      }),
+    },
+    killed,
+    kill: vi.fn(),
+    on: vi.fn().mockImplementation(function (
+      this: MockChildProcess,
+      event: string,
+      callback: (...args: any[]) => void,
+    ) {
+      // Check for custom event handlers first
+      if (customEventHandlers?.[event]) {
+        customEventHandlers[event](callback);
+        return mockProcess;
+      }
+
+      // Default event handling
+      if (event === 'error' && error) {
+        // Error event fires immediately
+        setImmediate(() => callback(error));
+      } else if (event === 'close' && !error) {
+        // Close event fires after data events
+        setImmediate(() => callback(exitCode));
+      }
+      return mockProcess;
+    }),
+  };
+
+  return mockProcess;
+}
+
+/**
+ * Type assertion helper to cast mock to ChildProcess
+ */
+export function asMockChildProcess(mock: MockChildProcess): ChildProcess {
+  return mock as unknown as ChildProcess;
+}

test/util/modelAuditInstall.test.ts

@@ -0,0 +1,35 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getModelAuditCurrentVersion } from '../../src/updates';
+import { checkModelAuditInstalled } from '../../src/util/modelAuditInstall';
+
+vi.mock('../../src/updates', () => ({
+  getModelAuditCurrentVersion: vi.fn(),
+}));
+
+const mockedGetModelAuditCurrentVersion = vi.mocked(getModelAuditCurrentVersion);
+
+describe('checkModelAuditInstalled', () => {
+  beforeEach(() => {
+    mockedGetModelAuditCurrentVersion.mockReset();
+  });
+
+  afterEach(() => {
+    mockedGetModelAuditCurrentVersion.mockReset();
+  });
+
+  it.each([
+    '0.2.16',
+    '1.0.0',
+    '0.2.19',
+  ])('returns the installed version when modelaudit reports %s', async (version) => {
+    mockedGetModelAuditCurrentVersion.mockResolvedValue(version);
+
+    await expect(checkModelAuditInstalled()).resolves.toEqual({ installed: true, version });
+  });
+
+  it('returns installed false when modelaudit is not installed', async () => {
+    mockedGetModelAuditCurrentVersion.mockResolvedValue(null);
+
+    await expect(checkModelAuditInstalled()).resolves.toEqual({ installed: false, version: null });
+  });
+});

test/util/modelAuditResults.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+import {
+  getModelAuditVerdict,
+  parseCompleteModelAuditResults,
+} from '../../src/util/modelAuditResults';
+
+describe('modelAuditResults', () => {
+  it('parses complete scan results', () => {
+    const results = parseCompleteModelAuditResults({
+      total_checks: 2,
+      passed_checks: 1,
+      failed_checks: 1,
+      has_errors: false,
+      issues: [],
+      checks: [
+        { name: 'safe', status: 'passed', message: 'ok' },
+        { name: 'pickle', status: 'failed', message: 'failed' },
+      ],
+    });
+
+    expect(results.failed_checks).toBe(1);
+    expect(getModelAuditVerdict(results)).toEqual({
+      hasFindings: true,
+      hasErrors: true,
+      hasFailedChecks: true,
+    });
+  });
+
+  it('keeps informational-only results clean', () => {
+    expect(
+      getModelAuditVerdict({
+        has_errors: false,
+        issues: [{ severity: 'info', message: 'metadata only' }],
+        checks: [],
+      }),
+    ).toEqual({
+      hasFindings: false,
+      hasErrors: false,
+      hasFailedChecks: false,
+    });
+  });
+});

test/util/namedMetrics.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, it } from 'vitest';
+import { accumulateNamedMetric, backfillNamedScoreWeights } from '../../src/util/namedMetrics';
+
+describe('accumulateNamedMetric', () => {
+  it('preserves weighted totals from grading results while keeping assertion counts', () => {
+    const metrics = {
+      namedScores: {},
+      namedScoresCount: {},
+      namedScoreWeights: {},
+    };
+
+    accumulateNamedMetric(metrics, {
+      metricName: 'accuracy',
+      metricValue: 0.75,
+      gradingResult: {
+        pass: false,
+        score: 0.75,
+        reason: 'weighted metric',
+        namedScoreWeights: { accuracy: 4 },
+        componentResults: [
+          {
+            pass: true,
+            score: 1,
+            reason: 'critical passed',
+            assertion: { type: 'contains', value: 'critical', metric: 'accuracy' },
+          },
+          {
+            pass: false,
+            score: 0,
+            reason: 'optional failed',
+            assertion: { type: 'contains', value: 'missing', metric: 'accuracy' },
+          },
+        ],
+      },
+    });
+
+    expect(metrics).toEqual({
+      namedScores: { accuracy: 3 },
+      namedScoresCount: { accuracy: 2 },
+      namedScoreWeights: { accuracy: 4 },
+    });
+  });
+
+  it('falls back to rendered assertion counts when stored weights are absent', () => {
+    const metrics = {
+      namedScores: {},
+      namedScoresCount: {},
+      namedScoreWeights: {},
+    };
+
+    accumulateNamedMetric(metrics, {
+      metricName: 'accuracy:alpha',
+      metricValue: 0.8,
+      testVars: { suffix: 'alpha' },
+      gradingResult: {
+        pass: true,
+        score: 0.8,
+        reason: 'templated metric',
+        componentResults: [
+          {
+            pass: true,
+            score: 0.8,
+            reason: 'templated metric',
+            assertion: { type: 'contains', value: 'alpha', metric: 'accuracy:{{ suffix }}' },
+          },
+        ],
+      },
+    });
+
+    expect(metrics).toEqual({
+      namedScores: { 'accuracy:alpha': 0.8 },
+      namedScoresCount: { 'accuracy:alpha': 1 },
+      namedScoreWeights: { 'accuracy:alpha': 1 },
+    });
+  });
+});
+
+describe('backfillNamedScoreWeights', () => {
+  it('fills missing weights from assertion counts without overwriting existing weights', () => {
+    const metrics = {
+      namedScores: { accuracy: 1.6, safety: 0.8 },
+      namedScoresCount: { accuracy: 2, safety: 1 },
+      namedScoreWeights: { accuracy: 4 },
+    };
+
+    backfillNamedScoreWeights(metrics);
+
+    expect(metrics).toEqual({
+      namedScores: { accuracy: 1.6, safety: 0.8 },
+      namedScoresCount: { accuracy: 2, safety: 1 },
+      namedScoreWeights: { accuracy: 4, safety: 1 },
+    });
+  });
+
+  it('initializes missing weights from assertion counts for legacy metrics', () => {
+    const metrics = {
+      namedScores: { accuracy: 1.6 },
+      namedScoresCount: { accuracy: 2 },
+    };
+
+    backfillNamedScoreWeights(metrics);
+
+    expect(metrics).toEqual({
+      namedScores: { accuracy: 1.6 },
+      namedScoresCount: { accuracy: 2 },
+      namedScoreWeights: { accuracy: 2 },
+    });
+  });
+});

test/util/oauth.test.ts

@@ -0,0 +1,281 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const mockFetchWithProxy = vi.hoisted(() => vi.fn());
+vi.mock('../../src/util/fetch/index', () => ({
+  fetchWithProxy: mockFetchWithProxy,
+}));
+
+import { fetchOAuthToken, TOKEN_REFRESH_BUFFER_MS } from '../../src/util/oauth';
+
+describe('oauth utils', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockFetchWithProxy.mockReset();
+  });
+
+  describe('TOKEN_REFRESH_BUFFER_MS', () => {
+    it('should be 60 seconds', () => {
+      expect(TOKEN_REFRESH_BUFFER_MS).toBe(60000);
+    });
+  });
+
+  describe('fetchOAuthToken', () => {
+    it('should fetch token with client_credentials grant', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'test-token',
+          expires_in: 3600,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      const result = await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'client_credentials',
+        clientId: 'test-client',
+        clientSecret: 'test-secret',
+      });
+
+      expect(result.accessToken).toBe('test-token');
+      expect(result.expiresAt).toBeGreaterThan(Date.now());
+
+      expect(mockFetchWithProxy).toHaveBeenCalledWith('https://auth.example.com/token', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        body: expect.stringContaining('grant_type=client_credentials'),
+      });
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      expect(callBody).toContain('client_id=test-client');
+      expect(callBody).toContain('client_secret=test-secret');
+    });
+
+    it('should fetch token with password grant', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'password-token',
+          expires_in: 7200,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      const result = await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'password',
+        clientId: 'test-client',
+        clientSecret: 'test-secret',
+        username: 'testuser',
+        password: 'testpass',
+      });
+
+      expect(result.accessToken).toBe('password-token');
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      expect(callBody).toContain('grant_type=password');
+      expect(callBody).toContain('username=testuser');
+      expect(callBody).toContain('password=testpass');
+    });
+
+    it('should include scopes when provided', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'scoped-token',
+          expires_in: 3600,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'client_credentials',
+        clientId: 'test-client',
+        scopes: ['read', 'write', 'admin'],
+      });
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      expect(callBody).toContain('scope=read+write+admin');
+    });
+
+    it('should work without optional clientId and clientSecret', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'minimal-token',
+          expires_in: 3600,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      const result = await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'client_credentials',
+      });
+
+      expect(result.accessToken).toBe('minimal-token');
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      expect(callBody).not.toContain('client_id');
+      expect(callBody).not.toContain('client_secret');
+    });
+
+    it('should default expires_in to 3600 seconds when not provided', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'no-expiry-token',
+          // No expires_in field
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      const beforeFetch = Date.now();
+      const result = await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'client_credentials',
+      });
+
+      // Should default to 3600 seconds (1 hour)
+      const expectedMinExpiry = beforeFetch + 3600 * 1000;
+      const expectedMaxExpiry = Date.now() + 3600 * 1000;
+      expect(result.expiresAt).toBeGreaterThanOrEqual(expectedMinExpiry);
+      expect(result.expiresAt).toBeLessThanOrEqual(expectedMaxExpiry);
+    });
+
+    it('should use provided expires_in value', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'custom-expiry-token',
+          expires_in: 1800, // 30 minutes
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      const beforeFetch = Date.now();
+      const result = await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'client_credentials',
+      });
+
+      // Should use 1800 seconds (30 minutes)
+      const expectedMinExpiry = beforeFetch + 1800 * 1000;
+      const expectedMaxExpiry = Date.now() + 1800 * 1000;
+      expect(result.expiresAt).toBeGreaterThanOrEqual(expectedMinExpiry);
+      expect(result.expiresAt).toBeLessThanOrEqual(expectedMaxExpiry);
+    });
+
+    it('should throw error when response is not ok', async () => {
+      const mockResponse = {
+        ok: false,
+        status: 401,
+        statusText: 'Unauthorized',
+        text: vi.fn().mockResolvedValue('{"error":"invalid_client"}'),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      await expect(
+        fetchOAuthToken({
+          tokenUrl: 'https://auth.example.com/token',
+          grantType: 'client_credentials',
+          clientId: 'bad-client',
+        }),
+      ).rejects.toThrow('OAuth token request failed with status 401 Unauthorized');
+    });
+
+    it('should throw error when access_token is missing in response', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          // No access_token field
+          token_type: 'Bearer',
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      await expect(
+        fetchOAuthToken({
+          tokenUrl: 'https://auth.example.com/token',
+          grantType: 'client_credentials',
+        }),
+      ).rejects.toThrow('OAuth token response missing access_token');
+    });
+
+    it('should handle password grant without username/password gracefully', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'token-without-creds',
+          expires_in: 3600,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      // Password grant without username/password - the function should still make the request
+      // (server will likely reject it, but that's not our concern here)
+      const result = await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'password',
+      });
+
+      expect(result.accessToken).toBe('token-without-creds');
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      expect(callBody).toContain('grant_type=password');
+      // Should not have username= or password= fields (distinct from grant_type=password)
+      expect(callBody).not.toContain('username=');
+      expect(callBody).not.toContain('&password=');
+    });
+
+    it('should not include empty scopes array', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'no-scope-token',
+          expires_in: 3600,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'client_credentials',
+        scopes: [],
+      });
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      expect(callBody).not.toContain('scope');
+    });
+
+    it('should URL-encode special characters in credentials', async () => {
+      const mockResponse = {
+        ok: true,
+        json: vi.fn().mockResolvedValue({
+          access_token: 'encoded-token',
+          expires_in: 3600,
+        }),
+      };
+      mockFetchWithProxy.mockResolvedValue(mockResponse);
+
+      await fetchOAuthToken({
+        tokenUrl: 'https://auth.example.com/token',
+        grantType: 'password',
+        clientId: 'client+id',
+        clientSecret: 'secret&value',
+        username: 'user@example.com',
+        password: 'pass=word!',
+      });
+
+      const callBody = mockFetchWithProxy.mock.calls[0][1].body;
+      // URLSearchParams should encode special characters
+      expect(callBody).toContain('client_id=client%2Bid');
+      expect(callBody).toContain('client_secret=secret%26value');
+      expect(callBody).toContain('username=user%40example.com');
+      expect(callBody).toContain('password=pass%3Dword%21');
+    });
+  });
+});

test/util/objectUtils.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it } from 'vitest';
+import { removeEmpty } from '../../src/util/objectUtils';
+
+describe('objectUtils', () => {
+  describe('removeEmpty', () => {
+    it('removes empty arrays from the top level', () => {
+      const input = {
+        name: 'test',
+        emptyArray: [],
+        nonEmptyArray: [1, 2, 3],
+      };
+
+      const result = removeEmpty(input);
+
+      expect(result).toEqual({
+        name: 'test',
+        nonEmptyArray: [1, 2, 3],
+      });
+      expect(result.emptyArray).toBeUndefined();
+    });
+
+    it('removes empty objects from the top level', () => {
+      const input = {
+        name: 'test',
+        emptyObj: {},
+        nonEmptyObj: { key: 'value' },
+      };
+
+      const result = removeEmpty(input);
+
+      expect(result).toEqual({
+        name: 'test',
+        nonEmptyObj: { key: 'value' },
+      });
+      expect(result.emptyObj).toBeUndefined();
+    });
+
+    it('does not remove empty arrays or objects in nested objects', () => {
+      const input = {
+        level1: {
+          emptyArr: [],
+          emptyObj: {},
+          validKey: 'value',
+        },
+      };
+
+      const result = removeEmpty(input);
+
+      expect(result).toEqual({
+        level1: {
+          emptyArr: [],
+          emptyObj: {},
+          validKey: 'value',
+        },
+      });
+    });
+
+    it('does not modify the original object', () => {
+      const original = {
+        name: 'test',
+        emptyArray: [],
+        nested: { empty: {} },
+      };
+
+      const originalCopy = JSON.parse(JSON.stringify(original));
+      removeEmpty(original);
+
+      expect(original).toEqual(originalCopy);
+    });
+  });
+});

test/util/outputFormats.test.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it } from 'vitest';
+import { getOutputFileFormat, SUPPORTED_OUTPUT_FILE_FORMATS } from '../../src/util/outputFormats';
+
+describe('outputFormats', () => {
+  it('detects JUnit XML outputs by the full suffix', () => {
+    expect(getOutputFileFormat('results.junit.xml')).toBe('junit.xml');
+    expect(getOutputFileFormat('reports/ci.RESULTS.JUNIT.XML')).toBe('junit.xml');
+  });
+
+  it('keeps Promptfoo XML separate from JUnit XML routing', () => {
+    expect(getOutputFileFormat('results.xml')).toBe('xml');
+    expect(getOutputFileFormat('results.junit')).toBeUndefined();
+  });
+
+  it('advertises junit.xml as a supported output format', () => {
+    expect(SUPPORTED_OUTPUT_FILE_FORMATS).toContain('junit.xml');
+  });
+});

test/util/packageImportErrors.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from 'vitest';
+import { isMissingPackageImportError } from '../../src/util/packageImportErrors';
+
+describe('isMissingPackageImportError', () => {
+  it('recognizes missing optional packages', () => {
+    const error = Object.assign(new Error('Cannot find package @googleapis/sheets'), {
+      code: 'ERR_MODULE_NOT_FOUND',
+    });
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(true);
+  });
+
+  it('recognizes CommonJS-style missing optional package errors', () => {
+    const error = Object.assign(new Error('Missing optional dependency @googleapis/sheets'), {
+      code: 'MODULE_NOT_FOUND',
+    });
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(true);
+  });
+
+  it('ignores import failures for unrelated packages', () => {
+    const error = new Error('Cannot find package @promptfoo/unrelated');
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(false);
+  });
+
+  it('preserves import-time runtime errors from present packages', () => {
+    const error = new Error('@googleapis/sheets failed during initialization');
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(false);
+  });
+
+  it('recognizes the optional package itself from a realistic ESM message', () => {
+    const error = Object.assign(
+      new Error(
+        "Cannot find package '@googleapis/sheets' imported from /app/dist/src/googleSheets.js",
+      ),
+      { code: 'ERR_MODULE_NOT_FOUND' },
+    );
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(true);
+  });
+
+  it('recognizes a subpath import of the optional package', () => {
+    const error = Object.assign(
+      new Error(
+        "Cannot find package '@googleapis/sheets/build/index' imported from /app/dist/src/googleSheets.js",
+      ),
+      { code: 'ERR_MODULE_NOT_FOUND' },
+    );
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(true);
+  });
+
+  it('does not misreport a missing transitive dependency as the optional package (ESM)', () => {
+    // The optional package is installed, but one of ITS dependencies is not.
+    // The package name appears only in the importer path, not as the failed
+    // specifier, so it must not be reported as the package itself being absent.
+    const error = Object.assign(
+      new Error(
+        "Cannot find package 'gaxios' imported from /app/node_modules/@googleapis/sheets/build/index.js",
+      ),
+      { code: 'ERR_MODULE_NOT_FOUND' },
+    );
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(false);
+  });
+
+  it('does not misreport a missing transitive dependency as the optional package (CommonJS)', () => {
+    const error = Object.assign(
+      new Error(
+        "Cannot find module 'gaxios'\nRequire stack:\n- /app/node_modules/@googleapis/sheets/build/index.js",
+      ),
+      { code: 'MODULE_NOT_FOUND' },
+    );
+
+    expect(isMissingPackageImportError(error, '@googleapis/sheets')).toBe(false);
+  });
+});

test/util/pathUtils.test.ts

@@ -0,0 +1,95 @@
+import path from 'path';
+
+import { describe, expect, it } from 'vitest';
+import { safeJoin, safeResolve } from '../../src/util/pathUtils';
+
+/**
+ * Helper to create file:// URLs in a cross-platform way
+ */
+function getFileUrl(path: string): string {
+  return process.platform === 'win32'
+    ? `file:///C:/${path.replace(/\\/g, '/')}`
+    : `file:///${path}`;
+}
+
+describe('pathUtils', () => {
+  describe('safeResolve', () => {
+    it('returns absolute path unchanged', () => {
+      const absolutePath = path.resolve('/absolute/path/file.txt');
+      expect(safeResolve('some/base/path', absolutePath)).toBe(absolutePath);
+    });
+
+    it('returns file URL unchanged', () => {
+      const fileUrl = getFileUrl('absolute/path/file.txt');
+      expect(safeResolve('some/base/path', fileUrl)).toBe(fileUrl);
+    });
+
+    it('returns Windows file URL unchanged', () => {
+      const windowsFileUrl = 'file://C:/path/file.txt';
+      expect(safeResolve('some/base/path', windowsFileUrl)).toBe(windowsFileUrl);
+    });
+
+    it('returns HTTP URLs unchanged', () => {
+      const httpUrl = 'https://example.com/file.txt';
+      expect(safeResolve('some/base/path', httpUrl)).toBe(httpUrl);
+    });
+
+    it('resolves relative paths', () => {
+      const expected = path.resolve('base/path', 'relative/file.txt');
+      expect(safeResolve('base/path', 'relative/file.txt')).toBe(expected);
+    });
+
+    it('handles multiple path segments', () => {
+      const absolutePath = path.resolve('/absolute/path/file.txt');
+      expect(safeResolve('base', 'path', absolutePath)).toBe(absolutePath);
+
+      const expected = path.resolve('base', 'path', 'relative/file.txt');
+      expect(safeResolve('base', 'path', 'relative/file.txt')).toBe(expected);
+    });
+
+    it('handles empty input', () => {
+      expect(safeResolve()).toBe(path.resolve());
+      expect(safeResolve('')).toBe(path.resolve(''));
+    });
+  });
+
+  describe('safeJoin', () => {
+    it('returns absolute path unchanged', () => {
+      const absolutePath = path.resolve('/absolute/path/file.txt');
+      expect(safeJoin('some/base/path', absolutePath)).toBe(absolutePath);
+    });
+
+    it('returns file URL unchanged', () => {
+      const fileUrl = getFileUrl('absolute/path/file.txt');
+      expect(safeJoin('some/base/path', fileUrl)).toBe(fileUrl);
+    });
+
+    it('returns Windows file URL unchanged', () => {
+      const windowsFileUrl = 'file://C:/path/file.txt';
+      expect(safeJoin('some/base/path', windowsFileUrl)).toBe(windowsFileUrl);
+    });
+
+    it('returns HTTP URLs unchanged', () => {
+      const httpUrl = 'https://example.com/file.txt';
+      expect(safeJoin('some/base/path', httpUrl)).toBe(httpUrl);
+    });
+
+    it('joins relative paths', () => {
+      const expected = path.join('base/path', 'relative/file.txt');
+      expect(safeJoin('base/path', 'relative/file.txt')).toBe(expected);
+    });
+
+    it('handles multiple path segments', () => {
+      const absolutePath = path.resolve('/absolute/path/file.txt');
+      expect(safeJoin('base', 'path', absolutePath)).toBe(absolutePath);
+
+      const expected = path.join('base', 'path', 'relative/file.txt');
+      expect(safeJoin('base', 'path', 'relative/file.txt')).toBe(expected);
+    });
+
+    it('handles empty input', () => {
+      expect(safeJoin()).toBe(path.join());
+      expect(safeJoin('')).toBe(path.join(''));
+    });
+  });
+});

test/util/processShim.test.ts

@@ -0,0 +1,181 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import {
+  _createBrowserProcessShim,
+  _isBrowserEnvironment,
+  getProcessShim,
+} from '../../src/util/processShim';
+
+describe('processShim', () => {
+  describe('_isBrowserEnvironment', () => {
+    const originalWindow = globalThis.window;
+    const originalSelf = globalThis.self;
+
+    afterEach(() => {
+      vi.resetAllMocks();
+      // Restore original globals
+      if (originalWindow === undefined) {
+        delete (globalThis as Record<string, unknown>).window;
+      } else {
+        (globalThis as Record<string, unknown>).window = originalWindow;
+      }
+      if (originalSelf === undefined) {
+        delete (globalThis as Record<string, unknown>).self;
+      } else {
+        (globalThis as Record<string, unknown>).self = originalSelf;
+      }
+    });
+
+    it('should return false in Node.js environment', () => {
+      // In Node.js, process.versions.node exists
+      expect(_isBrowserEnvironment()).toBe(false);
+    });
+
+    it('should return false even when window is defined in Node.js (jsdom/happy-dom)', () => {
+      // This tests the fix for test environments that define window
+      (globalThis as Record<string, unknown>).window = {};
+
+      // Should still return false because process.versions.node exists
+      expect(_isBrowserEnvironment()).toBe(false);
+    });
+
+    it('should return false even when self.importScripts is defined in Node.js', () => {
+      // Test environments might also mock web worker globals
+      (globalThis as Record<string, unknown>).self = {
+        importScripts: () => {},
+      };
+
+      // Should still return false because process.versions.node exists
+      expect(_isBrowserEnvironment()).toBe(false);
+    });
+  });
+
+  describe('_createBrowserProcessShim', () => {
+    afterEach(() => {
+      vi.resetAllMocks();
+    });
+
+    it('should return an object with env property', () => {
+      const shim = _createBrowserProcessShim();
+
+      expect(shim.env).toBeDefined();
+      expect(typeof shim.env).toBe('object');
+    });
+
+    it('should return an object with mainModule property', () => {
+      const shim = _createBrowserProcessShim();
+
+      expect(shim.mainModule).toBeDefined();
+    });
+
+    it('should throw an error when mainModule.require is called', () => {
+      const shim = _createBrowserProcessShim();
+
+      expect(() => {
+        shim.mainModule!.require('fs');
+      }).toThrow('require() is not available in browser transforms');
+    });
+
+    it('should have mainModule with expected structure', () => {
+      const shim = _createBrowserProcessShim();
+      const mainModule = shim.mainModule!;
+
+      expect(mainModule.exports).toEqual({});
+      expect(mainModule.id).toBe('.');
+      expect(mainModule.filename).toBe('');
+      expect(mainModule.loaded).toBe(true);
+      expect(mainModule.children).toEqual([]);
+      expect(mainModule.paths).toEqual([]);
+    });
+  });
+
+  describe('getProcessShim', () => {
+    const originalWindow = globalThis.window;
+
+    afterEach(() => {
+      vi.resetAllMocks();
+      // Restore original globals
+      if (originalWindow === undefined) {
+        delete (globalThis as Record<string, unknown>).window;
+      } else {
+        (globalThis as Record<string, unknown>).window = originalWindow;
+      }
+    });
+
+    it('should return a process shim in Node.js environment', () => {
+      const shim = getProcessShim();
+
+      expect(shim).toBeDefined();
+      expect(shim.mainModule).toBeDefined();
+    });
+
+    it('should return a working require function in Node.js environment', () => {
+      const shim = getProcessShim();
+
+      // In Node.js, mainModule.require should work
+      const path = shim.mainModule!.require('path');
+      expect(path.join).toBeDefined();
+      expect(typeof path.join).toBe('function');
+    });
+
+    it('should return Node.js shim even when window is defined (jsdom/happy-dom)', () => {
+      // This tests that we correctly detect Node.js in test environments
+      (globalThis as Record<string, unknown>).window = {};
+
+      const shim = getProcessShim();
+
+      // Should still work because process.versions.node exists
+      expect(shim).toBeDefined();
+      const path = shim.mainModule!.require('path');
+      expect(path.join).toBeDefined();
+    });
+
+    it('should cache the Node.js shim for subsequent calls', () => {
+      const shim1 = getProcessShim();
+      const shim2 = getProcessShim();
+
+      // Should return the same cached instance
+      expect(shim1).toBe(shim2);
+    });
+  });
+
+  describe('getProcessShim integration with transforms', () => {
+    afterEach(() => {
+      vi.resetAllMocks();
+    });
+
+    it('should allow inline transforms to access process shim', () => {
+      const shim = getProcessShim();
+
+      // Create an inline transform function like the actual code does
+      const transformFn = new Function(
+        'data',
+        'context',
+        'process',
+        'return data.toUpperCase()',
+      ) as (data: string, context: object, process: typeof global.process) => string;
+
+      const result = transformFn('hello', {}, shim);
+
+      expect(result).toBe('HELLO');
+    });
+
+    it('should allow inline transforms to use process.mainModule.require in Node.js', () => {
+      const shim = getProcessShim();
+
+      // Create a transform that uses require
+      const transformFn = new Function(
+        'data',
+        'context',
+        'process',
+        `
+        const path = process.mainModule.require('path');
+        return path.basename(data);
+        `,
+      ) as (data: string, context: object, process: typeof global.process) => string;
+
+      const result = transformFn('/foo/bar/test.txt', {}, shim);
+
+      expect(result).toBe('test.txt');
+    });
+  });
+});

test/util/promptfooCommand.test.ts

@@ -0,0 +1,190 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import {
+  detectInstaller,
+  isRunningUnderNpx,
+  promptfooCommand,
+} from '../../src/util/promptfooCommand';
+import { mockProcessEnv } from './utils';
+
+describe('nextCommand', () => {
+  let originalEnv: NodeJS.ProcessEnv;
+
+  beforeEach(() => {
+    originalEnv = { ...process.env };
+    mockProcessEnv({ ...originalEnv }, { clear: true });
+  });
+
+  afterEach(() => {
+    mockProcessEnv(originalEnv, { clear: true });
+  });
+
+  describe('detectInstaller', () => {
+    it('should detect npx from npm_execpath', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      expect(detectInstaller()).toBe('npx');
+    });
+
+    it('should detect npx from npm_lifecycle_script', () => {
+      mockProcessEnv({ npm_lifecycle_script: 'npx some-command' });
+      expect(detectInstaller()).toBe('npx');
+    });
+
+    it('should detect npx from process.execPath', () => {
+      const originalExecPath = process.execPath;
+      Object.defineProperty(process, 'execPath', {
+        value: '/path/to/npx/node',
+        configurable: true,
+      });
+
+      expect(detectInstaller()).toBe('npx');
+
+      Object.defineProperty(process, 'execPath', {
+        value: originalExecPath,
+        configurable: true,
+      });
+    });
+
+    it('should detect npx from npm_config_user_agent', () => {
+      mockProcessEnv({ npm_config_user_agent: 'npx/10.5.0 npm/10.8.2 node/v18.20.4' });
+      expect(detectInstaller()).toBe('npx');
+    });
+
+    it('should detect brew from npm_config_prefix', () => {
+      mockProcessEnv({ npm_config_prefix: '/usr/local/Homebrew/Cellar/node/20.0.0/bin' });
+      expect(detectInstaller()).toBe('brew');
+    });
+
+    it('should detect brew from process.execPath', () => {
+      const originalExecPath = process.execPath;
+      Object.defineProperty(process, 'execPath', {
+        value: '/usr/local/Homebrew/bin/node',
+        configurable: true,
+      });
+
+      expect(detectInstaller()).toBe('brew');
+
+      Object.defineProperty(process, 'execPath', {
+        value: originalExecPath,
+        configurable: true,
+      });
+    });
+
+    it('should detect npm-global from npm_config_user_agent', () => {
+      mockProcessEnv({ npm_config_user_agent: 'npm/10.8.2 node/v18.20.4' });
+      expect(detectInstaller()).toBe('npm-global');
+    });
+
+    it('should return unknown for unrecognized patterns', () => {
+      mockProcessEnv({ npm_config_user_agent: 'yarn/1.22.0' });
+      expect(detectInstaller()).toBe('unknown');
+    });
+
+    it('should prioritize npm_execpath over user agent', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      mockProcessEnv({ npm_config_user_agent: 'npm/10.8.2 node/v18.20.4' });
+      expect(detectInstaller()).toBe('npx');
+    });
+
+    it('should prioritize npm_lifecycle_script over user agent', () => {
+      mockProcessEnv({ npm_lifecycle_script: 'npx some-command' });
+      mockProcessEnv({ npm_config_user_agent: 'npm/10.8.2 node/v18.20.4' });
+      expect(detectInstaller()).toBe('npx');
+    });
+
+    it('should handle empty environment variables', () => {
+      mockProcessEnv({ npm_execpath: undefined });
+      mockProcessEnv({ npm_lifecycle_script: undefined });
+      mockProcessEnv({ npm_config_prefix: undefined });
+      mockProcessEnv({ npm_config_user_agent: undefined });
+      expect(detectInstaller()).toBe('unknown');
+    });
+
+    it('should handle case insensitive homebrew detection', () => {
+      mockProcessEnv({ npm_config_prefix: '/usr/local/homebrew/cellar/node/20.0.0/bin' });
+      expect(detectInstaller()).toBe('brew');
+    });
+
+    it('should detect brew with Windows-style paths', () => {
+      mockProcessEnv({ npm_config_prefix: 'C:\\Users\\user\\homebrew\\Cellar\\node\\20.0.0\\bin' });
+      expect(detectInstaller()).toBe('brew');
+    });
+  });
+
+  describe('promptfooCommand', () => {
+    it('should return npx command for npx installer', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      expect(promptfooCommand('eval')).toBe('npx promptfoo@latest eval');
+    });
+
+    it('should return regular command for brew installer', () => {
+      mockProcessEnv({ npm_config_prefix: '/usr/local/Homebrew/Cellar/node/20.0.0/bin' });
+      expect(promptfooCommand('eval')).toBe('promptfoo eval');
+    });
+
+    it('should return regular command for npm-global installer', () => {
+      mockProcessEnv({ npm_config_user_agent: 'npm/10.8.2 node/v18.20.4' });
+      expect(promptfooCommand('eval')).toBe('promptfoo eval');
+    });
+
+    it('should return regular command for unknown installer', () => {
+      mockProcessEnv({ npm_config_user_agent: 'yarn/1.22.0' });
+      expect(promptfooCommand('eval')).toBe('promptfoo eval');
+    });
+
+    it('should handle complex subcommands', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      expect(promptfooCommand('redteam init')).toBe('npx promptfoo@latest redteam init');
+    });
+
+    it('should handle subcommands with flags', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      expect(promptfooCommand('eval -c config.yaml')).toBe(
+        'npx promptfoo@latest eval -c config.yaml',
+      );
+    });
+
+    it('should handle empty subcommand for npx', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      expect(promptfooCommand('')).toBe('npx promptfoo@latest');
+    });
+
+    it('should handle empty subcommand for non-npx', () => {
+      mockProcessEnv({ npm_config_prefix: '/usr/local/Homebrew/Cellar/node/20.0.0/bin' });
+      expect(promptfooCommand('')).toBe('promptfoo');
+    });
+  });
+
+  describe('isRunningUnderNpx (legacy compatibility)', () => {
+    it('should return true for npx installer', () => {
+      mockProcessEnv({ npm_execpath: '/path/to/npx' });
+      expect(isRunningUnderNpx()).toBe(true);
+    });
+
+    it('should return false for non-npx installers', () => {
+      mockProcessEnv({ npm_config_prefix: '/usr/local/Homebrew/Cellar/node/20.0.0/bin' });
+      expect(isRunningUnderNpx()).toBe(false);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle partial matches in strings', () => {
+      mockProcessEnv({ npm_execpath: '/some/path/with-npx-in-middle/but-not-npx' });
+      expect(detectInstaller()).toBe('npx'); // Should still match because contains 'npx'
+    });
+
+    it('should handle npm_config_user_agent with multiple patterns', () => {
+      mockProcessEnv({ npm_config_user_agent: 'npx/10.5.0 npm/10.8.2 node/v18.20.4' });
+      expect(detectInstaller()).toBe('npx'); // Should detect npx, not npm
+    });
+
+    it('should handle undefined environment variables gracefully', () => {
+      mockProcessEnv({ npm_execpath: undefined });
+      mockProcessEnv({ npm_lifecycle_script: undefined });
+      mockProcessEnv({ npm_config_prefix: undefined });
+      mockProcessEnv({ npm_config_user_agent: undefined });
+
+      expect(() => detectInstaller()).not.toThrow();
+      expect(detectInstaller()).toBe('unknown');
+    });
+  });
+});

test/util/provider.test.ts

@@ -0,0 +1,620 @@
+import * as path from 'path';
+
+import { describe, expect, it, vi } from 'vitest';
+import {
+  checkProviderApiKeys,
+  doesProviderRefMatch,
+  getProviderDescription,
+  getProviderIdentifier,
+  isAnthropicProvider,
+  isGoogleProvider,
+  isOpenAiProvider,
+  isProviderAllowed,
+  providerToIdentifier,
+  sanitizeProviderIdForLog,
+} from '../../src/util/provider';
+import {
+  canonicalizeProviderId,
+  isProviderConfigFileReference,
+  normalizeProviderRef,
+} from '../../src/util/providerRef';
+import { createMockProvider } from '../factories/provider';
+
+import type { ApiProvider } from '../../src/types/index';
+
+describe('normalizeProviderRef', () => {
+  it('identifies provider config file references without treating script providers as config', () => {
+    expect(isProviderConfigFileReference('file://providers.yaml')).toBe(true);
+    expect(isProviderConfigFileReference('file://providers.yml')).toBe(true);
+    expect(isProviderConfigFileReference('file://providers.json')).toBe(true);
+    expect(isProviderConfigFileReference('file://provider.js')).toBe(false);
+  });
+
+  it('normalizes ProviderOptionsMap refs with explicit id overrides', () => {
+    const descriptor = normalizeProviderRef({
+      'openai:responses:gpt-5.4': {
+        id: 'custom-openai',
+        label: 'Fast OpenAI',
+        config: { temperature: 0.2 },
+      },
+    });
+
+    expect(descriptor).toMatchObject({
+      kind: 'map',
+      id: 'custom-openai',
+      label: 'Fast OpenAI',
+      loadProviderPath: 'openai:responses:gpt-5.4',
+      loadOptions: {
+        id: 'custom-openai',
+        label: 'Fast OpenAI',
+        config: { temperature: 0.2 },
+      },
+    });
+  });
+
+  it('does not treat ProviderOptions-shaped objects as ProviderOptionsMap refs', () => {
+    expect(normalizeProviderRef({ config: { temperature: 0.2 } })).toMatchObject({
+      kind: 'unknown',
+      id: 'unknown',
+    });
+    expect(normalizeProviderRef({ prompts: ['prompt1'] })).toMatchObject({
+      kind: 'unknown',
+      id: 'unknown',
+    });
+    expect(normalizeProviderRef({ config: { id: 'openai:gpt-4' } })).toMatchObject({
+      kind: 'unknown',
+      id: 'unknown',
+    });
+  });
+
+  it('requires ProviderOptionsMap refs to use a single provider id key', () => {
+    expect(
+      normalizeProviderRef({
+        'openai:gpt-4': { config: { temperature: 0.2 } },
+        'anthropic:messages:claude-sonnet-4-5': { config: { temperature: 0.1 } },
+      }),
+    ).toMatchObject({
+      kind: 'unknown',
+      id: 'unknown',
+    });
+  });
+
+  it('guards every ProviderOptions key against ProviderOptionsMap misclassification', () => {
+    // Each key from the ProviderOptions interface must be guarded so that
+    // objects like { transform: "..." } are not mistaken for { "transform": { ...providerOpts } }.
+    // If you add a key to ProviderOptions, add it here too.
+    const providerOptionKeys = [
+      'id',
+      'label',
+      'config',
+      'prompts',
+      'transform',
+      'delay',
+      'env',
+      'inputs',
+    ];
+    for (const key of providerOptionKeys) {
+      const obj = { [key]: { nested: true } };
+      const descriptor = normalizeProviderRef(obj);
+      expect(descriptor.kind, `key "${key}" should not produce kind 'map'`).not.toBe('map');
+    }
+  });
+
+  it('uses function labels before positional custom-function fallbacks', () => {
+    const labeled = Object.assign(async () => ({ output: 'ok' }), { label: 'custom-label' });
+    const unlabeled = async () => ({ output: 'ok' });
+
+    expect(normalizeProviderRef(labeled, { index: 3 })).toMatchObject({
+      kind: 'function',
+      id: 'custom-label',
+      label: 'custom-label',
+    });
+    expect(normalizeProviderRef(unlabeled, { index: 3 })).toMatchObject({
+      kind: 'function',
+      id: 'custom-function-3',
+    });
+  });
+
+  it('falls back to labels for malformed provider configs used by filtering', () => {
+    expect(normalizeProviderRef({ id: '', label: 'Provider2' }, { index: 1 })).toMatchObject({
+      kind: 'unknown',
+      id: 'Provider2',
+      label: 'Provider2',
+    });
+  });
+
+  it('classifies string providers as named vs file based on extension', () => {
+    expect(normalizeProviderRef('openai:responses:gpt-5.4')).toMatchObject({
+      kind: 'named',
+      id: 'openai:responses:gpt-5.4',
+      loadProviderPath: 'openai:responses:gpt-5.4',
+    });
+    expect(normalizeProviderRef('file://providers.yaml')).toMatchObject({
+      kind: 'file',
+      id: 'file://providers.yaml',
+      loadProviderPath: 'file://providers.yaml',
+    });
+    expect(normalizeProviderRef('file://provider.js')).toMatchObject({
+      kind: 'named',
+      id: 'file://provider.js',
+      loadProviderPath: 'file://provider.js',
+    });
+  });
+
+  it('normalizes ProviderOptionsMap without nested id override (key becomes id)', () => {
+    const descriptor = normalizeProviderRef({
+      'openai:responses:gpt-5.4': {
+        config: { temperature: 0.2 },
+      },
+    });
+
+    expect(descriptor).toMatchObject({
+      kind: 'map',
+      id: 'openai:responses:gpt-5.4',
+      loadProviderPath: 'openai:responses:gpt-5.4',
+      loadOptions: {
+        id: 'openai:responses:gpt-5.4',
+        config: { temperature: 0.2 },
+      },
+    });
+  });
+
+  it('returns kind unknown for empty string provider IDs', () => {
+    expect(normalizeProviderRef('')).toMatchObject({ kind: 'unknown', id: 'unknown' });
+    expect(normalizeProviderRef('', { index: 2 })).toMatchObject({
+      kind: 'unknown',
+      id: 'unknown-2',
+    });
+  });
+
+  it('returns kind unknown for null, undefined, and non-provider types', () => {
+    expect(normalizeProviderRef(null)).toMatchObject({ kind: 'unknown', id: 'unknown' });
+    expect(normalizeProviderRef(undefined)).toMatchObject({ kind: 'unknown', id: 'unknown' });
+    expect(normalizeProviderRef(42)).toMatchObject({ kind: 'unknown', id: 'unknown' });
+    expect(normalizeProviderRef([])).toMatchObject({ kind: 'unknown', id: 'unknown' });
+    expect(normalizeProviderRef([{ id: 'openai:gpt-4' }])).toMatchObject({
+      kind: 'unknown',
+      id: 'unknown',
+    });
+  });
+});
+
+describe('canonicalizeProviderId', () => {
+  it('resolves relative file:// paths to absolute', () => {
+    const cwd = process.cwd();
+    expect(canonicalizeProviderId('file://./provider.js')).toBe(
+      `file://${path.join(cwd, 'provider.js')}`,
+    );
+  });
+
+  it('preserves absolute file:// paths', () => {
+    expect(canonicalizeProviderId('file:///absolute/path.js')).toBe('file:///absolute/path.js');
+  });
+
+  it('resolves exec: paths with slashes', () => {
+    const cwd = process.cwd();
+    expect(canonicalizeProviderId('exec:./script.py')).toBe(`exec:${path.join(cwd, 'script.py')}`);
+  });
+
+  it('preserves exec: paths without slashes', () => {
+    expect(canonicalizeProviderId('exec:my-script')).toBe('exec:my-script');
+  });
+
+  it('resolves python: paths with slashes', () => {
+    const cwd = process.cwd();
+    expect(canonicalizeProviderId('python:./provider.py')).toBe(
+      `python:${path.join(cwd, 'provider.py')}`,
+    );
+  });
+
+  it('resolves golang: paths with slashes', () => {
+    const cwd = process.cwd();
+    expect(canonicalizeProviderId('golang:./main.go')).toBe(`golang:${path.join(cwd, 'main.go')}`);
+  });
+
+  it('preserves golang: paths without slashes', () => {
+    expect(canonicalizeProviderId('golang:my-binary')).toBe('golang:my-binary');
+  });
+
+  it('wraps bare .js/.ts/.mjs paths with file://', () => {
+    const cwd = process.cwd();
+    expect(canonicalizeProviderId('./provider.js')).toBe(`file://${path.join(cwd, 'provider.js')}`);
+    expect(canonicalizeProviderId('./provider.ts')).toBe(`file://${path.join(cwd, 'provider.ts')}`);
+    expect(canonicalizeProviderId('./provider.mjs')).toBe(
+      `file://${path.join(cwd, 'provider.mjs')}`,
+    );
+  });
+
+  it('does not wrap bare .js/.ts/.mjs without path separators', () => {
+    expect(canonicalizeProviderId('provider.js')).toBe('provider.js');
+  });
+
+  it('passes through plain provider IDs unchanged', () => {
+    expect(canonicalizeProviderId('openai:responses:gpt-5.4')).toBe('openai:responses:gpt-5.4');
+    expect(canonicalizeProviderId('echo')).toBe('echo');
+  });
+});
+
+describe('providerToIdentifier', () => {
+  it('works with provider string', () => {
+    expect(providerToIdentifier('gpt-3.5-turbo')).toStrictEqual('gpt-3.5-turbo');
+  });
+
+  it('works with provider id undefined', () => {
+    expect(providerToIdentifier(undefined)).toBeUndefined();
+  });
+
+  it('works with ApiProvider', () => {
+    const providerId = 'custom';
+    const apiProvider = {
+      id() {
+        return providerId;
+      },
+      callApi: vi.fn(),
+    } as ApiProvider;
+
+    expect(providerToIdentifier(apiProvider)).toStrictEqual(providerId);
+  });
+
+  it('works with ProviderOptions', () => {
+    const providerId = 'custom';
+    const providerOptions = {
+      id: providerId,
+    };
+
+    expect(providerToIdentifier(providerOptions)).toStrictEqual(providerId);
+  });
+
+  it('uses label when present on ProviderOptions', () => {
+    const providerOptions = {
+      id: 'file://provider.js',
+      label: 'my-provider',
+    };
+
+    expect(providerToIdentifier(providerOptions)).toStrictEqual('my-provider');
+  });
+
+  it('canonicalizes relative file paths to absolute', () => {
+    const originalCwd = process.cwd();
+    expect(providerToIdentifier('file://./provider.js')).toStrictEqual(
+      `file://${path.join(originalCwd, 'provider.js')}`,
+    );
+  });
+
+  it('canonicalizes JavaScript files without file:// prefix', () => {
+    const originalCwd = process.cwd();
+    expect(providerToIdentifier('./provider.js')).toStrictEqual(
+      `file://${path.join(originalCwd, 'provider.js')}`,
+    );
+  });
+
+  it('preserves absolute file paths', () => {
+    expect(providerToIdentifier('file:///absolute/path/provider.js')).toStrictEqual(
+      'file:///absolute/path/provider.js',
+    );
+  });
+
+  it('canonicalizes exec: paths', () => {
+    const originalCwd = process.cwd();
+    expect(providerToIdentifier('exec:./script.py')).toStrictEqual(
+      `exec:${path.join(originalCwd, 'script.py')}`,
+    );
+  });
+
+  it('canonicalizes python: paths', () => {
+    const originalCwd = process.cwd();
+    expect(providerToIdentifier('python:./provider.py')).toStrictEqual(
+      `python:${path.join(originalCwd, 'provider.py')}`,
+    );
+  });
+});
+
+describe('getProviderIdentifier', () => {
+  it('returns label when present', () => {
+    const provider = createMockProvider({ id: 'openai:gpt-4', label: 'my-custom-label' });
+    expect(getProviderIdentifier(provider)).toBe('my-custom-label');
+  });
+
+  it('returns id when no label', () => {
+    const provider = createMockProvider({ id: 'openai:gpt-4' });
+    expect(getProviderIdentifier(provider)).toBe('openai:gpt-4');
+  });
+});
+
+describe('getProviderDescription', () => {
+  it('returns both label and id when both present', () => {
+    const provider = createMockProvider({ id: 'openai:gpt-4', label: 'my-custom-label' });
+    expect(getProviderDescription(provider)).toBe('my-custom-label (openai:gpt-4)');
+  });
+
+  it('returns only id when no label', () => {
+    const provider = createMockProvider({ id: 'openai:gpt-4' });
+    expect(getProviderDescription(provider)).toBe('openai:gpt-4');
+  });
+
+  it('returns only id when label equals id', () => {
+    const provider = createMockProvider({ id: 'openai:gpt-4', label: 'openai:gpt-4' });
+    expect(getProviderDescription(provider)).toBe('openai:gpt-4');
+  });
+});
+
+describe('sanitizeProviderIdForLog', () => {
+  it('redacts credentials in URL provider IDs', () => {
+    expect(
+      sanitizeProviderIdForLog(
+        'http://example.com/api?cursor=sk-123456789012345678901234567890&safe=ok',
+      ),
+    ).toBe('http://example.com/api?cursor=%5BREDACTED%5D&safe=ok');
+  });
+
+  it('leaves non-URL provider IDs unchanged', () => {
+    expect(sanitizeProviderIdForLog('openai:gpt-4.1')).toBe('openai:gpt-4.1');
+  });
+
+  it('preserves provider descriptions while sanitizing URL IDs', () => {
+    expect(
+      sanitizeProviderIdForLog(
+        'http://example.com/api?cursor=sk-123456789012345678901234567890 (HttpProvider)',
+      ),
+    ).toBe('http://example.com/api?cursor=%5BREDACTED%5D (HttpProvider)');
+  });
+});
+
+describe('doesProviderRefMatch', () => {
+  const createProvider = (id: string, label?: string): ApiProvider =>
+    createMockProvider({ id, label });
+
+  it('matches exact label', () => {
+    const provider = createProvider('openai:gpt-4', 'fast-model');
+    expect(doesProviderRefMatch('fast-model', provider)).toBe(true);
+    expect(doesProviderRefMatch('slow-model', provider)).toBe(false);
+  });
+
+  it('matches exact id', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(doesProviderRefMatch('openai:gpt-4', provider)).toBe(true);
+    expect(doesProviderRefMatch('openai:gpt-3.5', provider)).toBe(false);
+  });
+
+  it('matches wildcard on id', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(doesProviderRefMatch('openai:*', provider)).toBe(true);
+    expect(doesProviderRefMatch('anthropic:*', provider)).toBe(false);
+  });
+
+  it('matches wildcard on label', () => {
+    const provider = createProvider('openai:gpt-4', 'openai-fast');
+    expect(doesProviderRefMatch('openai-*', provider)).toBe(true);
+    expect(doesProviderRefMatch('anthropic-*', provider)).toBe(false);
+  });
+
+  it('matches legacy prefix on id', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(doesProviderRefMatch('openai', provider)).toBe(true);
+    expect(doesProviderRefMatch('anthropic', provider)).toBe(false);
+  });
+
+  it('matches legacy prefix on label', () => {
+    const provider = createProvider('custom-provider', 'openai:custom');
+    expect(doesProviderRefMatch('openai', provider)).toBe(true);
+  });
+
+  it('does not match partial strings', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(doesProviderRefMatch('openai:gpt', provider)).toBe(false);
+    expect(doesProviderRefMatch('gpt-4', provider)).toBe(false);
+  });
+
+  it('matches canonicalized file paths', () => {
+    // Provider has relative path, ref uses absolute - should still match after canonicalization
+    const absolutePath = path.resolve('./my-provider.js');
+    const provider = createProvider(`file://${absolutePath}`);
+    expect(doesProviderRefMatch('./my-provider.js', provider)).toBe(true);
+    expect(doesProviderRefMatch(`file://${absolutePath}`, provider)).toBe(true);
+  });
+});
+
+describe('isProviderAllowed', () => {
+  const createProvider = (id: string, label?: string): ApiProvider =>
+    createMockProvider({ id, label });
+
+  it('allows all providers when no filter', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(isProviderAllowed(provider, undefined)).toBe(true);
+  });
+
+  it('blocks all providers with empty array', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(isProviderAllowed(provider, [])).toBe(false);
+  });
+
+  it('allows matching providers', () => {
+    const provider = createProvider('openai:gpt-4', 'fast-model');
+    expect(isProviderAllowed(provider, ['fast-model'])).toBe(true);
+    expect(isProviderAllowed(provider, ['openai:gpt-4'])).toBe(true);
+    expect(isProviderAllowed(provider, ['openai:*'])).toBe(true);
+  });
+
+  it('blocks non-matching providers', () => {
+    const provider = createProvider('openai:gpt-4', 'fast-model');
+    expect(isProviderAllowed(provider, ['slow-model'])).toBe(false);
+    expect(isProviderAllowed(provider, ['anthropic:*'])).toBe(false);
+  });
+
+  it('allows if any filter matches', () => {
+    const provider = createProvider('openai:gpt-4');
+    expect(isProviderAllowed(provider, ['anthropic:*', 'openai:*'])).toBe(true);
+    expect(isProviderAllowed(provider, ['anthropic:*', 'google:*'])).toBe(false);
+  });
+});
+
+describe('isOpenAiProvider', () => {
+  it('detects direct OpenAI providers', () => {
+    expect(isOpenAiProvider('openai:chat:gpt-4o')).toBe(true);
+    expect(isOpenAiProvider('openai:gpt-4')).toBe(true);
+    expect(isOpenAiProvider('openai:completion:gpt-3.5-turbo')).toBe(true);
+    expect(isOpenAiProvider('openai:embedding:text-embedding-ada-002')).toBe(true);
+  });
+
+  it('detects Azure OpenAI providers', () => {
+    // azureopenai: is always OpenAI
+    expect(isOpenAiProvider('azureopenai:chat:my-deployment')).toBe(true);
+    // azure: with OpenAI model indicators
+    expect(isOpenAiProvider('azure:chat:gpt-4-deployment')).toBe(true);
+    expect(isOpenAiProvider('azure:chat:my-gpt-35-turbo')).toBe(true);
+    expect(isOpenAiProvider('azure:completion:davinci-002')).toBe(true);
+    expect(isOpenAiProvider('azure:embedding:text-embedding-ada-002')).toBe(true);
+  });
+
+  it('does not match Azure without OpenAI model indicators', () => {
+    expect(isOpenAiProvider('azure:chat:my-custom-deployment')).toBe(false);
+    expect(isOpenAiProvider('azure:foundry-agent:my-agent')).toBe(false);
+  });
+
+  it('is case insensitive', () => {
+    expect(isOpenAiProvider('OpenAI:chat:gpt-4')).toBe(true);
+    expect(isOpenAiProvider('AZUREOPENAI:chat:my-deployment')).toBe(true);
+    expect(isOpenAiProvider('AZURE:chat:GPT-4-deployment')).toBe(true);
+  });
+
+  it('does not match non-OpenAI providers', () => {
+    expect(isOpenAiProvider('anthropic:messages:claude-3-5-sonnet')).toBe(false);
+    expect(isOpenAiProvider('google:gemini-pro')).toBe(false);
+    expect(isOpenAiProvider('bedrock:anthropic.claude-3-5-sonnet')).toBe(false);
+    expect(isOpenAiProvider('vertex:gemini-2.0-flash')).toBe(false);
+  });
+});
+
+describe('isAnthropicProvider', () => {
+  it('detects direct Anthropic providers', () => {
+    expect(isAnthropicProvider('anthropic:messages:claude-3-5-sonnet')).toBe(true);
+    expect(isAnthropicProvider('anthropic:completion:claude-2.1')).toBe(true);
+    expect(isAnthropicProvider('anthropic:claude-opus-4-6')).toBe(true);
+    expect(isAnthropicProvider('anthropic:claude-opus-4-5-20251101')).toBe(true);
+  });
+
+  it('detects Bedrock with Claude models', () => {
+    expect(isAnthropicProvider('bedrock:anthropic.claude-3-5-sonnet-20240620-v1:0')).toBe(true);
+    expect(isAnthropicProvider('bedrock:anthropic.claude-opus-4-6-v1')).toBe(true);
+    expect(isAnthropicProvider('bedrock:anthropic.claude-opus-4-5-20251101-v1:0')).toBe(true);
+    expect(isAnthropicProvider('bedrock:converse:anthropic.claude-3-opus-20240229-v1:0')).toBe(
+      true,
+    );
+    expect(isAnthropicProvider('bedrock:us.anthropic.claude-3-5-sonnet-20240620-v1:0')).toBe(true);
+    expect(isAnthropicProvider('bedrock:eu.anthropic.claude-3-haiku-20240307-v1:0')).toBe(true);
+  });
+
+  it('detects Vertex with Claude models', () => {
+    expect(isAnthropicProvider('vertex:claude-3-5-sonnet@20240620')).toBe(true);
+    expect(isAnthropicProvider('vertex:claude-3-opus@20240229')).toBe(true);
+  });
+
+  it('is case insensitive', () => {
+    expect(isAnthropicProvider('ANTHROPIC:messages:claude-3-5-sonnet')).toBe(true);
+    expect(isAnthropicProvider('BEDROCK:anthropic.CLAUDE-3-5-sonnet')).toBe(true);
+    expect(isAnthropicProvider('VERTEX:CLAUDE-3-opus')).toBe(true);
+  });
+
+  it('does not match non-Anthropic providers', () => {
+    expect(isAnthropicProvider('openai:chat:gpt-4o')).toBe(false);
+    expect(isAnthropicProvider('google:gemini-pro')).toBe(false);
+    expect(isAnthropicProvider('bedrock:amazon.titan-text-express-v1')).toBe(false);
+    expect(isAnthropicProvider('vertex:gemini-2.0-flash')).toBe(false);
+  });
+});
+
+describe('isGoogleProvider', () => {
+  it('detects direct Google AI Studio providers', () => {
+    expect(isGoogleProvider('google:gemini-pro')).toBe(true);
+    expect(isGoogleProvider('google:gemini-2.0-flash')).toBe(true);
+    expect(isGoogleProvider('google:live:gemini-2.0-flash')).toBe(true);
+    expect(isGoogleProvider('google:image:gemini-pro-vision')).toBe(true);
+  });
+
+  it('detects Vertex with Google models', () => {
+    expect(isGoogleProvider('vertex:gemini-2.0-flash')).toBe(true);
+    expect(isGoogleProvider('vertex:gemini-pro')).toBe(true);
+    expect(isGoogleProvider('vertex:text-bison')).toBe(true);
+    expect(isGoogleProvider('vertex:llama-3.3-70b')).toBe(true);
+  });
+
+  it('is case insensitive', () => {
+    expect(isGoogleProvider('GOOGLE:gemini-pro')).toBe(true);
+    expect(isGoogleProvider('VERTEX:GEMINI-2.0-flash')).toBe(true);
+  });
+
+  it('does not match Vertex with Claude models (they are Anthropic)', () => {
+    expect(isGoogleProvider('vertex:claude-3-5-sonnet@20240620')).toBe(false);
+    expect(isGoogleProvider('vertex:claude-3-opus@20240229')).toBe(false);
+  });
+
+  it('does not match non-Google providers', () => {
+    expect(isGoogleProvider('openai:chat:gpt-4o')).toBe(false);
+    expect(isGoogleProvider('anthropic:messages:claude-3-5-sonnet')).toBe(false);
+    expect(isGoogleProvider('bedrock:anthropic.claude-3-5-sonnet')).toBe(false);
+  });
+});
+
+describe('checkProviderApiKeys', () => {
+  const providerWithKey = (id: string, extras: Record<string, unknown> = {}): ApiProvider =>
+    Object.assign(createMockProvider({ id, config: {} }), {
+      getApiKey: () => undefined,
+      requiresApiKey: () => true,
+      ...extras,
+    }) as unknown as ApiProvider;
+
+  it('detects missing API key and maps to correct env var', () => {
+    const provider = providerWithKey('openai:gpt-4');
+
+    const result = checkProviderApiKeys([provider]);
+    expect(result.size).toBe(1);
+    expect(result.get('OPENAI_API_KEY')).toEqual(['openai:gpt-4']);
+  });
+
+  it('uses descriptions only when requested for user-facing error output', () => {
+    const provider = providerWithKey('openai:gpt-4', { label: 'primary' });
+
+    expect(checkProviderApiKeys([provider]).get('OPENAI_API_KEY')).toEqual(['openai:gpt-4']);
+    expect(
+      checkProviderApiKeys([provider], { useDescriptions: true }).get('OPENAI_API_KEY'),
+    ).toEqual(['primary (openai:gpt-4)']);
+  });
+
+  it('skips providers with valid key, no getApiKey method, or requiresApiKey false', () => {
+    const providers: ApiProvider[] = [
+      providerWithKey('openai:gpt-4', { getApiKey: () => 'sk-1234' }),
+      createMockProvider({ id: 'http:custom' }),
+      providerWithKey('openai:gpt-4', { requiresApiKey: () => false }),
+      Object.assign(
+        createMockProvider({ id: 'litellm:gpt-4', config: { apiKeyRequired: false } }),
+        { getApiKey: () => undefined },
+      ) as unknown as ApiProvider,
+    ];
+
+    const result = checkProviderApiKeys(providers);
+    expect(result.size).toBe(0);
+  });
+
+  it('skips azure providers (Azure AD token auth)', () => {
+    const provider = providerWithKey('azure:gpt-4');
+
+    const result = checkProviderApiKeys([provider]);
+    expect(result.size).toBe(0);
+  });
+
+  it('deduplicates multiple providers sharing the same env var', () => {
+    const providers: ApiProvider[] = [
+      providerWithKey('openai:gpt-4'),
+      providerWithKey('openai:gpt-5-mini'),
+      Object.assign(
+        createMockProvider({ id: 'anthropic:claude-sonnet-4-5-20250514', config: {} }),
+        { getApiKey: () => undefined },
+      ) as unknown as ApiProvider,
+    ];
+
+    const result = checkProviderApiKeys(providers);
+    expect(result.size).toBe(2);
+    expect(result.get('OPENAI_API_KEY')).toEqual(['openai:gpt-4', 'openai:gpt-5-mini']);
+    expect(result.get('ANTHROPIC_API_KEY')).toEqual(['anthropic:claude-sonnet-4-5-20250514']);
+  });
+});

test/util/providerResponse.test.ts

@@ -0,0 +1,150 @@
+import { describe, expect, it } from 'vitest';
+import { getActualPrompt, getActualPromptWithFallback } from '../../src/util/providerResponse';
+
+import type { ProviderResponse } from '../../src/types/providers';
+
+describe('getActualPrompt', () => {
+  it('should return undefined for undefined response', () => {
+    expect(getActualPrompt(undefined)).toBeUndefined();
+  });
+
+  it('should return undefined for response without prompt', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+    };
+    expect(getActualPrompt(response)).toBeUndefined();
+  });
+
+  it('should return string prompt directly', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: 'Hello, world!',
+    };
+    expect(getActualPrompt(response)).toBe('Hello, world!');
+  });
+
+  it('should return undefined for empty string prompt', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: '',
+    };
+    // Empty string is truthy check but empty, so we return undefined
+    expect(getActualPrompt(response)).toBeUndefined();
+  });
+
+  it('should stringify chat message array', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: [
+        { role: 'system', content: 'You are helpful' },
+        { role: 'user', content: 'Hello' },
+      ],
+    };
+    expect(getActualPrompt(response)).toBe(
+      '[{"role":"system","content":"You are helpful"},{"role":"user","content":"Hello"}]',
+    );
+  });
+
+  it('should format chat message array when formatted option is true', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: [
+        { role: 'system', content: 'You are helpful' },
+        { role: 'user', content: 'Hello' },
+      ],
+    };
+    const result = getActualPrompt(response, { formatted: true });
+    expect(result).toContain('\n');
+    expect(result).toContain('  ');
+  });
+
+  it('should return undefined for empty array prompt', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: [],
+    };
+    // Empty array is not useful
+    expect(getActualPrompt(response)).toBeUndefined();
+  });
+
+  it('should fall back to redteamFinalPrompt when prompt is not set', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      metadata: {
+        redteamFinalPrompt: 'fallback prompt',
+      },
+    };
+    expect(getActualPrompt(response)).toBe('fallback prompt');
+  });
+
+  it('should prioritize prompt over redteamFinalPrompt', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: 'provider prompt',
+      metadata: {
+        redteamFinalPrompt: 'fallback prompt',
+      },
+    };
+    expect(getActualPrompt(response)).toBe('provider prompt');
+  });
+
+  it('should not fall back to redteamFinalPrompt when prompt is empty string', () => {
+    // Empty string means provider explicitly set no prompt, so we don't fall back
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: '',
+      metadata: {
+        redteamFinalPrompt: 'fallback prompt',
+      },
+    };
+    // With empty string prompt explicitly set, we return undefined (not the fallback)
+    expect(getActualPrompt(response)).toBeUndefined();
+  });
+});
+
+describe('getActualPromptWithFallback', () => {
+  it('should return provider prompt when available', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: 'provider prompt',
+    };
+    expect(getActualPromptWithFallback(response, 'original prompt')).toBe('provider prompt');
+  });
+
+  it('should return redteamFinalPrompt when prompt not available', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      metadata: {
+        redteamFinalPrompt: 'redteam prompt',
+      },
+    };
+    expect(getActualPromptWithFallback(response, 'original prompt')).toBe('redteam prompt');
+  });
+
+  it('should return original prompt when neither prompt nor redteamFinalPrompt available', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+    };
+    expect(getActualPromptWithFallback(response, 'original prompt')).toBe('original prompt');
+  });
+
+  it('should return original prompt for undefined response', () => {
+    expect(getActualPromptWithFallback(undefined, 'original prompt')).toBe('original prompt');
+  });
+
+  it('should return original prompt when prompt is empty string', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: '',
+    };
+    expect(getActualPromptWithFallback(response, 'original prompt')).toBe('original prompt');
+  });
+
+  it('should return original prompt when prompt is empty array', () => {
+    const response: ProviderResponse = {
+      output: 'test output',
+      prompt: [],
+    };
+    expect(getActualPromptWithFallback(response, 'original prompt')).toBe('original prompt');
+  });
+});

test/util/readline.test.ts

@@ -0,0 +1,133 @@
+import readline from 'readline';
+
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { createReadlineInterface, promptUser, promptYesNo } from '../../src/util/readline';
+
+vi.mock('readline');
+
+vi.mock('../../src/util/readline', () => ({
+  createReadlineInterface: vi.fn(),
+  promptUser: vi.fn(),
+  promptYesNo: vi.fn(),
+}));
+
+describe('readline utils', () => {
+  let mockInterface: any;
+
+  beforeEach(() => {
+    mockInterface = {
+      question: vi.fn(),
+      close: vi.fn(),
+      on: vi.fn(),
+    };
+
+    vi.mocked(readline.createInterface).mockReturnValue(mockInterface);
+    vi.mocked(createReadlineInterface).mockReturnValue(mockInterface);
+    vi.mocked(promptUser).mockReset();
+    vi.mocked(promptYesNo).mockReset();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  afterAll(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe('createReadlineInterface', () => {
+    it('should create readline interface with stdin/stdout', () => {
+      const result = createReadlineInterface();
+
+      expect(createReadlineInterface).toHaveBeenCalledWith();
+      expect(result).toBe(mockInterface);
+    });
+  });
+
+  describe('promptUser', () => {
+    it('should resolve with user answer', async () => {
+      const question = 'Test question?';
+      const answer = 'Test answer';
+
+      vi.mocked(promptUser).mockResolvedValue(answer);
+
+      const result = await promptUser(question);
+      expect(result).toBe(answer);
+      expect(promptUser).toHaveBeenCalledWith(question);
+    });
+
+    it('should reject on error', async () => {
+      const error = new Error('Test error');
+
+      vi.mocked(promptUser).mockRejectedValue(error);
+
+      await expect(promptUser('Test question?')).rejects.toThrow(error);
+    });
+
+    it('should reject if readline creation fails', async () => {
+      const error = new Error('Creation failed');
+
+      vi.mocked(promptUser).mockRejectedValue(error);
+
+      await expect(promptUser('Test question?')).rejects.toThrow(error);
+    });
+  });
+
+  describe('promptYesNo', () => {
+    it('should return true for "y" with default no', async () => {
+      vi.mocked(promptYesNo).mockResolvedValue(true);
+
+      const result = await promptYesNo('Test question?', false);
+      expect(result).toBe(true);
+      expect(promptYesNo).toHaveBeenCalledWith('Test question?', false);
+    });
+
+    it('should return false for "n" with default yes', async () => {
+      vi.mocked(promptYesNo).mockResolvedValue(false);
+
+      const result = await promptYesNo('Test question?', true);
+      expect(result).toBe(false);
+      expect(promptYesNo).toHaveBeenCalledWith('Test question?', true);
+    });
+
+    it('should return default value for empty response', async () => {
+      vi.mocked(promptYesNo).mockResolvedValueOnce(true).mockResolvedValueOnce(false);
+
+      await expect(promptYesNo('Test question?', true)).resolves.toBe(true);
+      await expect(promptYesNo('Test question?', false)).resolves.toBe(false);
+    });
+
+    it('should handle different case inputs', async () => {
+      vi.mocked(promptYesNo).mockResolvedValueOnce(true).mockResolvedValueOnce(false);
+
+      await expect(promptYesNo('Test question?')).resolves.toBe(true);
+      await expect(promptYesNo('Test question?', true)).resolves.toBe(false);
+    });
+
+    it('should append correct suffix based on default value', async () => {
+      vi.mocked(promptYesNo).mockResolvedValue(true);
+
+      await promptYesNo('Test question?', true);
+      expect(promptYesNo).toHaveBeenCalledWith('Test question?', true);
+
+      await promptYesNo('Test question?', false);
+      expect(promptYesNo).toHaveBeenCalledWith('Test question?', false);
+    });
+
+    it('should return true for non-n input with defaultYes true', async () => {
+      vi.mocked(promptYesNo).mockResolvedValue(true);
+
+      const result = await promptYesNo('Test question?', true);
+      expect(result).toBe(true);
+      expect(promptYesNo).toHaveBeenCalledWith('Test question?', true);
+    });
+
+    it('should return false for input not starting with y with defaultYes false', async () => {
+      vi.mocked(promptYesNo).mockResolvedValue(false);
+
+      const result = await promptYesNo('Test question?', false);
+      expect(result).toBe(false);
+      expect(promptYesNo).toHaveBeenCalledWith('Test question?', false);
+    });
+  });
+});

test/util/redteamProbeLimit.test.ts

@@ -0,0 +1,327 @@
+import { randomUUID } from 'crypto';
+
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { evalResultsTable, evalsTable } from '../../src/database/tables';
+import { getUserEmail, isLoggedIntoCloud } from '../../src/globalConfig/accounts';
+import { runDbMigrations } from '../../src/migrate';
+import {
+  checkRedteamProbeLimit,
+  getMonthlyRedteamProbeUsage,
+  getMonthStartTimestamp,
+  MONTHLY_PROBE_LIMIT,
+} from '../../src/util/redteamProbeLimit';
+
+vi.mock('../../src/globalConfig/accounts', async () => {
+  const actual = await vi.importActual('../../src/globalConfig/accounts');
+  return {
+    ...actual,
+    isLoggedIntoCloud: vi.fn().mockReturnValue(false),
+    getUserEmail: vi.fn().mockReturnValue('test@example.com'),
+  };
+});
+
+/**
+ * Helper to insert a redteam eval with result rows directly into the database.
+ */
+async function insertRedteamEval(opts: {
+  createdAt?: number;
+  numResults?: number;
+  numRequestsPerResult?: number;
+}): Promise<string> {
+  const db = await getDb();
+  const evalId = randomUUID();
+  const createdAt = opts.createdAt ?? Date.now();
+  const numResults = opts.numResults ?? 1;
+  const numRequestsPerResult = opts.numRequestsPerResult ?? 1;
+
+  await db
+    .insert(evalsTable)
+    .values({
+      id: evalId,
+      createdAt,
+      config: { redteam: { plugins: [], strategies: [] } } as any,
+      results: {},
+      isRedteam: true,
+    })
+    .run();
+
+  for (let i = 0; i < numResults; i++) {
+    await db
+      .insert(evalResultsTable)
+      .values({
+        id: randomUUID(),
+        createdAt,
+        updatedAt: createdAt,
+        evalId,
+        promptIdx: 0,
+        testIdx: i,
+        testCase: { vars: {} },
+        prompt: { raw: 'test', label: 'test' },
+        provider: { id: 'test-provider' },
+        response: {
+          output: 'test output',
+          tokenUsage: {
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            numRequests: numRequestsPerResult,
+          },
+        },
+        success: true,
+        score: 1,
+      })
+      .run();
+  }
+
+  return evalId;
+}
+
+/**
+ * Helper to insert a non-redteam eval.
+ */
+async function insertNonRedteamEval(opts?: {
+  createdAt?: number;
+  numResults?: number;
+}): Promise<string> {
+  const db = await getDb();
+  const evalId = randomUUID();
+  const createdAt = opts?.createdAt ?? Date.now();
+  const numResults = opts?.numResults ?? 1;
+
+  await db
+    .insert(evalsTable)
+    .values({
+      id: evalId,
+      createdAt,
+      config: { providers: ['test'] } as any,
+      results: {},
+      isRedteam: false,
+    })
+    .run();
+
+  for (let i = 0; i < numResults; i++) {
+    await db
+      .insert(evalResultsTable)
+      .values({
+        id: randomUUID(),
+        createdAt,
+        updatedAt: createdAt,
+        evalId,
+        promptIdx: 0,
+        testIdx: i,
+        testCase: { vars: {} },
+        prompt: { raw: 'test', label: 'test' },
+        provider: { id: 'test-provider' },
+        response: {
+          output: 'test output',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, numRequests: 1 },
+        },
+        success: true,
+        score: 1,
+      })
+      .run();
+  }
+
+  return evalId;
+}
+
+describe('redteamProbeLimit', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(async () => {
+    const db = await getDb();
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals');
+
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date('2026-03-15T12:00:00.000Z'));
+    vi.mocked(isLoggedIntoCloud).mockReturnValue(false);
+    vi.mocked(getUserEmail).mockReturnValue('test@example.com');
+  });
+
+  afterEach(() => {
+    vi.runOnlyPendingTimers();
+    vi.useRealTimers();
+    vi.resetAllMocks();
+  });
+
+  describe('getMonthStartTimestamp', () => {
+    it('should return the first day of the current month at midnight', () => {
+      const result = getMonthStartTimestamp();
+      const date = new Date(result);
+      expect(date.getDate()).toBe(1);
+      expect(date.getHours()).toBe(0);
+      expect(date.getMinutes()).toBe(0);
+      expect(date.getSeconds()).toBe(0);
+      expect(date.getMilliseconds()).toBe(0);
+    });
+  });
+
+  describe('getMonthlyRedteamProbeUsage', () => {
+    it('should return 0 when there are no evals', async () => {
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(0);
+    });
+
+    it('should count probes from redteam evals in the current month', async () => {
+      await insertRedteamEval({ numResults: 3, numRequestsPerResult: 1 });
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(3);
+    });
+
+    it('should count multi-turn probes correctly using numRequests', async () => {
+      // Simulates a multi-turn strategy where each result has multiple target calls
+      await insertRedteamEval({ numResults: 2, numRequestsPerResult: 10 });
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(20);
+    });
+
+    it('should not count non-redteam evals', async () => {
+      await insertNonRedteamEval({ numResults: 5 });
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(0);
+    });
+
+    it('should not count evals from previous months', async () => {
+      await insertRedteamEval({
+        createdAt: getMonthStartTimestamp() - 1,
+        numResults: 5,
+        numRequestsPerResult: 1,
+      });
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(0);
+    });
+
+    it('should aggregate probes across multiple redteam evals', async () => {
+      await insertRedteamEval({ numResults: 3, numRequestsPerResult: 1 });
+      await insertRedteamEval({ numResults: 2, numRequestsPerResult: 5 });
+      // 3*1 + 2*5 = 13
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(13);
+    });
+
+    it('should fall back to 1 per result when numRequests is not present', async () => {
+      const db = await getDb();
+      const evalId = randomUUID();
+      const now = Date.now();
+
+      await db
+        .insert(evalsTable)
+        .values({
+          id: evalId,
+          createdAt: now,
+          config: { redteam: { plugins: [] } },
+          results: {},
+          isRedteam: true,
+        })
+        .run();
+
+      // Insert result without numRequests in tokenUsage
+      await db
+        .insert(evalResultsTable)
+        .values({
+          id: randomUUID(),
+          createdAt: now,
+          updatedAt: now,
+          evalId,
+          promptIdx: 0,
+          testIdx: 0,
+          testCase: { vars: {} },
+          prompt: { raw: 'test', label: 'test' },
+          provider: { id: 'test-provider' },
+          response: {
+            output: 'test output',
+            tokenUsage: { total: 10, prompt: 5, completion: 5 },
+          },
+          success: true,
+          score: 1,
+        })
+        .run();
+
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(1);
+    });
+
+    it('should detect redteam evals via config JSON even if isRedteam column is false', async () => {
+      const db = await getDb();
+      const evalId = randomUUID();
+      const now = Date.now();
+
+      // Insert with isRedteam=false but config has redteam key
+      await db
+        .insert(evalsTable)
+        .values({
+          id: evalId,
+          createdAt: now,
+          config: { redteam: { plugins: ['harmful'] } } as any,
+          results: {},
+          isRedteam: false,
+        })
+        .run();
+
+      await db
+        .insert(evalResultsTable)
+        .values({
+          id: randomUUID(),
+          createdAt: now,
+          updatedAt: now,
+          evalId,
+          promptIdx: 0,
+          testIdx: 0,
+          testCase: { vars: {} },
+          prompt: { raw: 'test', label: 'test' },
+          provider: { id: 'test-provider' },
+          response: {
+            output: 'test',
+            tokenUsage: { total: 10, prompt: 5, completion: 5, numRequests: 3 },
+          },
+          success: true,
+          score: 1,
+        })
+        .run();
+
+      await expect(getMonthlyRedteamProbeUsage()).resolves.toBe(3);
+    });
+  });
+
+  describe('checkRedteamProbeLimit', () => {
+    it('should return withinLimit=true when no usage', async () => {
+      const result = await checkRedteamProbeLimit();
+      expect(result.withinLimit).toBe(true);
+      expect(result.used).toBe(0);
+      expect(result.remaining).toBe(MONTHLY_PROBE_LIMIT);
+      expect(result.limit).toBe(MONTHLY_PROBE_LIMIT);
+    });
+
+    it('should return withinLimit=true when under limit', async () => {
+      await insertRedteamEval({ numResults: 10, numRequestsPerResult: 1 });
+      const result = await checkRedteamProbeLimit();
+      expect(result.withinLimit).toBe(true);
+      expect(result.used).toBe(10);
+      expect(result.remaining).toBe(MONTHLY_PROBE_LIMIT - 10);
+    });
+
+    it('should return withinLimit=false when at or over limit', async () => {
+      // Insert enough probes to exceed the limit
+      // We'll use a large numRequests to simulate hitting the limit
+      const numResultsNeeded = 1000;
+      const numRequestsPerResult = Math.ceil(MONTHLY_PROBE_LIMIT / numResultsNeeded) + 1;
+      await insertRedteamEval({ numResults: numResultsNeeded, numRequestsPerResult });
+
+      const result = await checkRedteamProbeLimit();
+      expect(result.withinLimit).toBe(false);
+      expect(result.used).toBeGreaterThanOrEqual(MONTHLY_PROBE_LIMIT);
+      expect(result.remaining).toBe(0);
+    });
+
+    it('should exempt cloud-authenticated users', async () => {
+      vi.mocked(isLoggedIntoCloud).mockReturnValue(true);
+
+      // Even with probes, cloud users should be exempt
+      await insertRedteamEval({ numResults: 10, numRequestsPerResult: 1 });
+
+      const result = await checkRedteamProbeLimit();
+      expect(result.withinLimit).toBe(true);
+      expect(result.remaining).toBe(Number.POSITIVE_INFINITY);
+      expect(result.limit).toBe(Number.POSITIVE_INFINITY);
+    });
+  });
+});

test/util/render.test.ts

@@ -0,0 +1,588 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { renderEnvOnlyInObject, renderVarsInObject } from '../../src/util/render';
+import { mockProcessEnv } from './utils';
+
+describe('renderVarsInObject', () => {
+  beforeEach(() => {
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: undefined });
+  });
+
+  afterEach(() => {
+    mockProcessEnv({ TEST_ENV_VAR: undefined });
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: undefined });
+  });
+
+  it('should render environment variables in objects', async () => {
+    mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+    const obj = { text: '{{ env.TEST_ENV_VAR }}' };
+    const rendered = renderVarsInObject(obj, {});
+    expect(rendered).toEqual({ text: 'env_value' });
+  });
+
+  it('should return object unchanged when no vars provided', async () => {
+    const obj = { text: '{{ variable }}', number: 42 };
+    const rendered = renderVarsInObject(obj);
+    expect(rendered).toEqual(obj);
+  });
+
+  it('should return object unchanged when vars is empty object', async () => {
+    const obj = { text: '{{ variable }}', number: 42 };
+    const rendered = renderVarsInObject(obj, {});
+    // Empty object {} is truthy, so templating still runs but with no variables
+    expect(rendered).toEqual({ text: '', number: 42 });
+  });
+
+  it('should return object unchanged when PROMPTFOO_DISABLE_TEMPLATING is true', async () => {
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    const obj = { text: '{{ variable }}' };
+    const vars = { variable: 'test_value' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toEqual(obj);
+  });
+
+  it('should render variables in string objects', async () => {
+    const obj = 'Hello {{ name }}!';
+    const vars = { name: 'World' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toBe('Hello World!');
+  });
+
+  it('should render variables in array objects', async () => {
+    const obj = ['{{ greeting }}', '{{ name }}', 42];
+    const vars = { greeting: 'Hello', name: 'World' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toEqual(['Hello', 'World', 42]);
+  });
+
+  it('should render variables in nested arrays', async () => {
+    const obj = [
+      ['{{ item1 }}', '{{ item2 }}'],
+      ['static', '{{ item3 }}'],
+    ];
+    const vars = { item1: 'first', item2: 'second', item3: 'third' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toEqual([
+      ['first', 'second'],
+      ['static', 'third'],
+    ]);
+  });
+
+  it('should render variables in nested objects', async () => {
+    const obj = {
+      level1: {
+        level2: {
+          text: '{{ variable }}',
+          number: 42,
+        },
+        array: ['{{ item }}'],
+      },
+    };
+    const vars = { variable: 'nested_value', item: 'array_item' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toEqual({
+      level1: {
+        level2: {
+          text: 'nested_value',
+          number: 42,
+        },
+        array: ['array_item'],
+      },
+    });
+  });
+
+  it('should handle function objects by calling them with vars', async () => {
+    const mockFunction = vi.fn().mockReturnValue({ result: '{{ value }}' });
+    const vars = { value: 'function_result' };
+    const rendered = renderVarsInObject(mockFunction, vars);
+
+    expect(mockFunction).toHaveBeenCalledWith({ vars });
+    // Function result is NOT recursively templated because vars is not passed in recursive call
+    expect(rendered).toEqual({ result: '{{ value }}' });
+  });
+
+  it('should handle null values', async () => {
+    const obj = null;
+    const vars = { variable: 'test' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toBeNull();
+  });
+
+  it('should handle undefined values', async () => {
+    const obj = undefined;
+    const vars = { variable: 'test' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toBeUndefined();
+  });
+
+  it('should handle primitive number values', async () => {
+    const obj = 42;
+    const vars = { variable: 'test' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toBe(42);
+  });
+
+  it('should handle primitive boolean values', async () => {
+    const obj = true;
+    const vars = { variable: 'test' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toBe(true);
+  });
+
+  it('should handle objects with null properties', async () => {
+    const obj = { nullProp: null, text: '{{ variable }}' };
+    const vars = { variable: 'test_value' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toEqual({ nullProp: null, text: 'test_value' });
+  });
+
+  it('should handle mixed type objects', async () => {
+    const obj = {
+      string: '{{ text }}',
+      number: 42,
+      boolean: true,
+      nullValue: null,
+      array: ['{{ item }}', 123],
+      nested: {
+        deep: '{{ deep_value }}',
+      },
+    };
+    const vars = { text: 'rendered', item: 'array_item', deep_value: 'deep_rendered' };
+    const rendered = renderVarsInObject(obj, vars);
+    expect(rendered).toEqual({
+      string: 'rendered',
+      number: 42,
+      boolean: true,
+      nullValue: null,
+      array: ['array_item', 123],
+      nested: {
+        deep: 'deep_rendered',
+      },
+    });
+  });
+
+  it('should handle function that returns complex object structure', async () => {
+    const complexFunction = vi.fn().mockReturnValue({
+      data: {
+        items: ['{{ item1 }}', '{{ item2 }}'],
+        metadata: { value: '{{ meta }}' },
+      },
+    });
+    const vars = { item1: 'first', item2: 'second', meta: 'metadata_value' };
+    const rendered = renderVarsInObject(complexFunction, vars);
+
+    expect(complexFunction).toHaveBeenCalledWith({ vars });
+    // Function result is NOT recursively templated because vars is not passed in recursive call
+    expect(rendered).toEqual({
+      data: {
+        items: ['{{ item1 }}', '{{ item2 }}'],
+        metadata: { value: '{{ meta }}' },
+      },
+    });
+  });
+});
+
+describe('renderEnvOnlyInObject', () => {
+  beforeEach(() => {
+    mockProcessEnv({ TEST_ENV_VAR: undefined });
+    mockProcessEnv({ AZURE_ENDPOINT: undefined });
+    mockProcessEnv({ API_VERSION: undefined });
+    mockProcessEnv({ PORT: undefined });
+    mockProcessEnv({ API_HOST: undefined });
+    mockProcessEnv({ BASE_URL: undefined });
+    mockProcessEnv({ EMPTY_VAR: undefined });
+    mockProcessEnv({ SPECIAL_CHARS: undefined });
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: undefined });
+  });
+
+  afterEach(() => {
+    mockProcessEnv({ TEST_ENV_VAR: undefined });
+    mockProcessEnv({ AZURE_ENDPOINT: undefined });
+    mockProcessEnv({ API_VERSION: undefined });
+    mockProcessEnv({ PORT: undefined });
+    mockProcessEnv({ API_HOST: undefined });
+    mockProcessEnv({ BASE_URL: undefined });
+    mockProcessEnv({ EMPTY_VAR: undefined });
+    mockProcessEnv({ SPECIAL_CHARS: undefined });
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: undefined });
+  });
+
+  describe('Basic rendering', () => {
+    it('should render simple dot notation env vars', async () => {
+      mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+      expect(renderEnvOnlyInObject('{{ env.TEST_ENV_VAR }}')).toBe('env_value');
+    });
+
+    it('should render bracket notation with single quotes', async () => {
+      mockProcessEnv({ ['VAR-WITH-DASH']: 'dash_value' });
+      expect(renderEnvOnlyInObject("{{ env['VAR-WITH-DASH'] }}")).toBe('dash_value');
+    });
+
+    it('should render bracket notation with double quotes', async () => {
+      mockProcessEnv({ ['VAR_NAME']: 'value' });
+      expect(renderEnvOnlyInObject('{{ env["VAR_NAME"] }}')).toBe('value');
+    });
+
+    it('should handle whitespace variations', async () => {
+      mockProcessEnv({ TEST: 'value' });
+      expect(renderEnvOnlyInObject('{{env.TEST}}')).toBe('value');
+      expect(renderEnvOnlyInObject('{{  env.TEST  }}')).toBe('value');
+      expect(renderEnvOnlyInObject('{{ env.TEST}}')).toBe('value');
+      expect(renderEnvOnlyInObject('{{env.TEST }}')).toBe('value');
+    });
+
+    it('should render empty string env vars', async () => {
+      mockProcessEnv({ EMPTY_VAR: '' });
+      expect(renderEnvOnlyInObject('{{ env.EMPTY_VAR }}')).toBe('');
+    });
+
+    it('should render env vars with special characters in value', async () => {
+      mockProcessEnv({ SPECIAL_CHARS: 'value with spaces & $pecial chars!' });
+      expect(renderEnvOnlyInObject('{{ env.SPECIAL_CHARS }}')).toBe(
+        'value with spaces & $pecial chars!',
+      );
+    });
+  });
+
+  describe('Filters and expressions (NEW functionality)', () => {
+    it('should support default filter with fallback', async () => {
+      mockProcessEnv({ EXISTING: 'exists' });
+      expect(renderEnvOnlyInObject("{{ env.EXISTING | default('fallback') }}")).toBe('exists');
+      // NEW: When env var doesn't exist but has default filter, Nunjucks renders it
+      expect(renderEnvOnlyInObject("{{ env.NONEXISTENT | default('fallback') }}")).toBe('fallback');
+    });
+
+    it('should support upper filter', async () => {
+      mockProcessEnv({ LOWERCASE: 'lowercase' });
+      expect(renderEnvOnlyInObject('{{ env.LOWERCASE | upper }}')).toBe('LOWERCASE');
+    });
+
+    it('should support lower filter', async () => {
+      mockProcessEnv({ UPPERCASE: 'UPPERCASE' });
+      expect(renderEnvOnlyInObject('{{ env.UPPERCASE | lower }}')).toBe('uppercase');
+    });
+
+    it('should support chained filters', async () => {
+      mockProcessEnv({ TEST: 'test' });
+      expect(renderEnvOnlyInObject("{{ env.TEST | default('x') | upper }}")).toBe('TEST');
+    });
+
+    it('should support complex filter expressions', async () => {
+      mockProcessEnv({ PORT: '8080' });
+      expect(renderEnvOnlyInObject('{{ env.PORT | int }}')).toBe('8080');
+    });
+
+    it('should handle filter with closing brace in argument', async () => {
+      mockProcessEnv({ VAR: 'value' });
+      // This is a tricky case: the default value contains }
+      expect(renderEnvOnlyInObject("{{ env.VAR | default('}') }}")).toBe('value');
+    });
+  });
+
+  describe('Preservation of non-env templates', () => {
+    it('should preserve vars templates', async () => {
+      mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+      expect(renderEnvOnlyInObject('{{ env.TEST_ENV_VAR }}, {{ vars.myVar }}')).toBe(
+        'env_value, {{ vars.myVar }}',
+      );
+    });
+
+    it('should preserve prompt templates', async () => {
+      mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+      expect(renderEnvOnlyInObject('{{ env.TEST_ENV_VAR }}, {{ prompt }}')).toBe(
+        'env_value, {{ prompt }}',
+      );
+    });
+
+    it('should preserve multiple non-env templates', async () => {
+      mockProcessEnv({ API_HOST: 'api.example.com' });
+      const template =
+        'Host: {{ env.API_HOST }}, Message: {{ vars.msg }}, Context: {{ context }}, Prompt: {{ prompt }}';
+      const expected =
+        'Host: api.example.com, Message: {{ vars.msg }}, Context: {{ context }}, Prompt: {{ prompt }}';
+      expect(renderEnvOnlyInObject(template)).toBe(expected);
+    });
+
+    it('should preserve templates with filters on non-env vars', async () => {
+      expect(renderEnvOnlyInObject("{{ vars.name | default('Guest') }}")).toBe(
+        "{{ vars.name | default('Guest') }}",
+      );
+    });
+
+    it('should preserve env templates in _conversation runtime vars', async () => {
+      mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+      const config = {
+        tests: [
+          {
+            vars: {
+              _conversation: [
+                {
+                  input: 'Tell me a secret',
+                  output: 'The answer is {{ env.TEST_ENV_VAR }}',
+                },
+              ],
+              regularVar: '{{ env.TEST_ENV_VAR }}',
+            },
+          },
+        ],
+      };
+
+      expect(renderEnvOnlyInObject(config)).toEqual({
+        tests: [
+          {
+            vars: {
+              _conversation: [
+                {
+                  input: 'Tell me a secret',
+                  output: 'The answer is {{ env.TEST_ENV_VAR }}',
+                },
+              ],
+              regularVar: 'env_value',
+            },
+          },
+        ],
+      });
+    });
+  });
+
+  describe('Undefined env vars', () => {
+    it('should preserve template if env var does not exist', async () => {
+      expect(renderEnvOnlyInObject('{{ env.NONEXISTENT }}')).toBe('{{ env.NONEXISTENT }}');
+    });
+
+    it('should preserve bracket notation if env var does not exist', async () => {
+      expect(renderEnvOnlyInObject("{{ env['MISSING'] }}")).toBe("{{ env['MISSING'] }}");
+    });
+  });
+
+  describe('Complex data structures', () => {
+    it('should work with nested objects', async () => {
+      mockProcessEnv({ LEVEL1: 'value1' });
+      mockProcessEnv({ LEVEL2: 'value2' });
+      const obj = {
+        level1: {
+          level2: {
+            env1: '{{ env.LEVEL1 }}',
+            env2: '{{ env.LEVEL2 }}',
+            vars: '{{ vars.test }}',
+          },
+        },
+      };
+      expect(renderEnvOnlyInObject(obj)).toEqual({
+        level1: {
+          level2: {
+            env1: 'value1',
+            env2: 'value2',
+            vars: '{{ vars.test }}',
+          },
+        },
+      });
+    });
+
+    it('should work with arrays', async () => {
+      mockProcessEnv({ ENV1: 'value1' });
+      mockProcessEnv({ ENV2: 'value2' });
+      const arr = ['{{ env.ENV1 }}', '{{ vars.test }}', '{{ env.ENV2 }}', 42];
+      expect(renderEnvOnlyInObject(arr)).toEqual(['value1', '{{ vars.test }}', 'value2', 42]);
+    });
+
+    it('should work with mixed nested structures', async () => {
+      mockProcessEnv({ API_KEY: 'secret123' });
+      const config = {
+        api: {
+          key: '{{ env.API_KEY }}',
+          endpoints: ['{{ env.BASE_URL }}/users', '{{ env.BASE_URL }}/posts'],
+        },
+        request: {
+          body: { message: '{{ vars.message }}' },
+          headers: { authorization: 'Bearer {{ env.API_KEY }}' },
+        },
+      };
+      const rendered = renderEnvOnlyInObject(config);
+      expect(rendered).toEqual({
+        api: {
+          key: 'secret123',
+          endpoints: ['{{ env.BASE_URL }}/users', '{{ env.BASE_URL }}/posts'],
+        },
+        request: {
+          body: { message: '{{ vars.message }}' },
+          headers: { authorization: 'Bearer secret123' },
+        },
+      });
+    });
+  });
+
+  describe('Primitive types', () => {
+    it('should handle null', async () => {
+      expect(renderEnvOnlyInObject(null)).toBeNull();
+    });
+
+    it('should handle undefined', async () => {
+      expect(renderEnvOnlyInObject(undefined)).toBeUndefined();
+    });
+
+    it('should handle numbers', async () => {
+      expect(renderEnvOnlyInObject(42)).toBe(42);
+    });
+
+    it('should handle booleans', async () => {
+      expect(renderEnvOnlyInObject(true)).toBe(true);
+      expect(renderEnvOnlyInObject(false)).toBe(false);
+    });
+  });
+
+  describe('Error handling', () => {
+    it('should preserve template on Nunjucks render error', async () => {
+      mockProcessEnv({ TEST: 'value' });
+      // Malformed filter that would cause Nunjucks error
+      const template = '{{ env.TEST | nonexistent_filter }}';
+      const rendered = renderEnvOnlyInObject(template);
+      // Should preserve the template if rendering fails
+      expect(rendered).toBe(template);
+    });
+  });
+
+  describe('PROMPTFOO_DISABLE_TEMPLATING flag', () => {
+    it('should return unchanged when flag is set', async () => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+      mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+      expect(renderEnvOnlyInObject('{{ env.TEST_ENV_VAR }}')).toBe('{{ env.TEST_ENV_VAR }}');
+    });
+
+    it('should return unchanged objects when flag is set', async () => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+      mockProcessEnv({ TEST: 'value' });
+      const obj = { key: '{{ env.TEST }}' };
+      expect(renderEnvOnlyInObject(obj)).toEqual({ key: '{{ env.TEST }}' });
+    });
+  });
+
+  describe('Real-world scenarios', () => {
+    it('should handle Azure provider config with mixed templates', async () => {
+      mockProcessEnv({ AZURE_ENDPOINT: 'test.openai.azure.com' });
+      mockProcessEnv({ API_VERSION: '2024-02-15' });
+      const config = {
+        apiHost: '{{ env.AZURE_ENDPOINT }}',
+        apiVersion: '{{ env.API_VERSION }}',
+        body: {
+          message: '{{ vars.userMessage }}',
+          user: '{{ vars.userId }}',
+        },
+      };
+      expect(renderEnvOnlyInObject(config)).toEqual({
+        apiHost: 'test.openai.azure.com',
+        apiVersion: '2024-02-15',
+        body: {
+          message: '{{ vars.userMessage }}',
+          user: '{{ vars.userId }}',
+        },
+      });
+    });
+
+    it('should handle HTTP provider with runtime vars', async () => {
+      mockProcessEnv({ BASE_URL: 'https://api.example.com' });
+      const config = {
+        url: '{{ env.BASE_URL }}/query',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: 'Bearer {{ env.API_TOKEN }}',
+        },
+        body: {
+          query: '{{ vars.userQuery }}',
+          context: '{{ vars.context }}',
+        },
+      };
+      const rendered = renderEnvOnlyInObject(config);
+      expect(rendered).toEqual({
+        url: 'https://api.example.com/query',
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: 'Bearer {{ env.API_TOKEN }}', // Undefined, preserved
+        },
+        body: {
+          query: '{{ vars.userQuery }}', // Runtime vars preserved
+          context: '{{ vars.context }}', // Runtime vars preserved
+        },
+      });
+    });
+
+    it('should handle complex provider config with filters', async () => {
+      mockProcessEnv({ API_HOST: 'api.example.com' });
+      mockProcessEnv({ PORT: '8080' });
+      const config = {
+        baseUrl: "{{ env.API_HOST | default('localhost') }}:{{ env.PORT }}",
+        timeout: '{{ env.TIMEOUT | default(30000) }}',
+        request: {
+          body: '{{ vars.payload }}',
+        },
+      };
+      const rendered = renderEnvOnlyInObject(config);
+      expect(rendered).toEqual({
+        baseUrl: 'api.example.com:8080',
+        timeout: '30000', // TIMEOUT undefined, but default filter renders it
+        request: {
+          body: '{{ vars.payload }}',
+        },
+      });
+    });
+  });
+
+  describe('Edge cases', () => {
+    it('should handle multiple env vars in same string', async () => {
+      mockProcessEnv({ HOST: 'example.com' });
+      mockProcessEnv({ PORT: '8080' });
+      expect(renderEnvOnlyInObject('{{ env.HOST }}:{{ env.PORT }}')).toBe('example.com:8080');
+    });
+
+    it('should handle env vars in middle of text', async () => {
+      mockProcessEnv({ NAME: 'World' });
+      expect(renderEnvOnlyInObject('Hello {{ env.NAME }}!')).toBe('Hello World!');
+    });
+
+    it('should handle templates with newlines', async () => {
+      mockProcessEnv({ VAR: 'value' });
+      expect(
+        renderEnvOnlyInObject(`Line 1: {{ env.VAR }}
+Line 2: {{ vars.test }}`),
+      ).toBe(`Line 1: value
+Line 2: {{ vars.test }}`);
+    });
+
+    it('should render env vars in strings longer than 50000 chars', async () => {
+      mockProcessEnv({ TEST_ENV_VAR: 'env_value' });
+      const padding = 'x'.repeat(60000);
+      const template = `${padding} {{ env.TEST_ENV_VAR }} ${padding}`;
+      expect(template.length).toBeGreaterThan(50000);
+      expect(renderEnvOnlyInObject(template)).toBe(`${padding} env_value ${padding}`);
+    });
+
+    it('should preserve non-env templates in long strings', async () => {
+      const padding = 'x'.repeat(60000);
+      const template = `${padding} {{ vars.myVar }} ${padding}`;
+      expect(renderEnvOnlyInObject(template)).toBe(template);
+    });
+
+    it('should handle long strings with mixed env and non-env templates', async () => {
+      mockProcessEnv({ HOST: 'example.com' });
+      const padding = 'x'.repeat(60000);
+      const template = `${padding} {{ env.HOST }} {{ vars.test }} ${padding}`;
+      expect(renderEnvOnlyInObject(template)).toBe(
+        `${padding} example.com {{ vars.test }} ${padding}`,
+      );
+    });
+
+    it('should handle long strings with no templates at all', async () => {
+      const longString = 'a'.repeat(100000);
+      expect(renderEnvOnlyInObject(longString)).toBe(longString);
+    });
+
+    it('should not confuse env in other contexts', async () => {
+      mockProcessEnv({ TEST: 'value' });
+      // Should not match "environment" or other words containing "env"
+      expect(renderEnvOnlyInObject('environment {{ vars.test }}')).toBe(
+        'environment {{ vars.test }}',
+      );
+    });
+  });
+});

test/util/runtime.test.ts

@@ -0,0 +1,134 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('../../src/envars', () => ({
+  getEnvString: vi.fn().mockReturnValue(undefined),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    info: vi.fn(),
+  },
+}));
+
+import { getEnvString } from '../../src/envars';
+import logger from '../../src/logger';
+import { isRunningUnderNpx, printBorder } from '../../src/util/runtime';
+
+describe('runtime utilities', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  describe('printBorder', () => {
+    it('should call logger.info with a border string', () => {
+      printBorder();
+
+      expect(logger.info).toHaveBeenCalledTimes(1);
+      const call = vi.mocked(logger.info).mock.calls[0][0];
+      expect(typeof call).toBe('string');
+      expect(call).toMatch(/^=+$/);
+    });
+
+    it('should print a border of repeated equals signs', () => {
+      printBorder();
+
+      const border = vi.mocked(logger.info).mock.calls[0][0] as string;
+      expect(border.length).toBeGreaterThan(0);
+      expect(border.split('').every((char) => char === '=')).toBe(true);
+    });
+  });
+
+  describe('isRunningUnderNpx', () => {
+    const originalExecPath = process.execPath;
+
+    afterEach(() => {
+      Object.defineProperty(process, 'execPath', {
+        value: originalExecPath,
+        writable: true,
+      });
+    });
+
+    it('should return false when not running under npx', () => {
+      vi.mocked(getEnvString).mockReturnValue(undefined as unknown as string);
+      Object.defineProperty(process, 'execPath', {
+        value: '/usr/local/bin/node',
+        writable: true,
+      });
+
+      expect(isRunningUnderNpx()).toBe(false);
+    });
+
+    it('should return true when npm_execpath contains npx', () => {
+      (vi.mocked(getEnvString) as any).mockImplementation((key: string) => {
+        if (key === 'npm_execpath') {
+          return '/usr/local/lib/node_modules/npm/bin/npx-cli.js';
+        }
+        return undefined;
+      });
+      Object.defineProperty(process, 'execPath', {
+        value: '/usr/local/bin/node',
+        writable: true,
+      });
+
+      expect(isRunningUnderNpx()).toBe(true);
+    });
+
+    it('should return true when process.execPath contains npx', () => {
+      vi.mocked(getEnvString).mockReturnValue(undefined as unknown as string);
+      Object.defineProperty(process, 'execPath', {
+        value: '/home/user/.npm/_npx/123/node_modules/.bin/node',
+        writable: true,
+      });
+
+      expect(isRunningUnderNpx()).toBe(true);
+    });
+
+    it('should return true when npm_lifecycle_script contains npx', () => {
+      (vi.mocked(getEnvString) as any).mockImplementation((key: string) => {
+        if (key === 'npm_lifecycle_script') {
+          return 'npx promptfoo eval';
+        }
+        return undefined;
+      });
+      Object.defineProperty(process, 'execPath', {
+        value: '/usr/local/bin/node',
+        writable: true,
+      });
+
+      expect(isRunningUnderNpx()).toBe(true);
+    });
+
+    it('should return false when env vars are empty strings', () => {
+      vi.mocked(getEnvString).mockReturnValue('');
+      Object.defineProperty(process, 'execPath', {
+        value: '/usr/local/bin/node',
+        writable: true,
+      });
+
+      expect(isRunningUnderNpx()).toBe(false);
+    });
+
+    it('should check npm_execpath first', () => {
+      (vi.mocked(getEnvString) as any).mockImplementation((key: string) => {
+        if (key === 'npm_execpath') {
+          return '/path/to/npx';
+        }
+        if (key === 'npm_lifecycle_script') {
+          return 'some other script';
+        }
+        return undefined;
+      });
+      Object.defineProperty(process, 'execPath', {
+        value: '/usr/local/bin/node',
+        writable: true,
+      });
+
+      expect(isRunningUnderNpx()).toBe(true);
+      expect(getEnvString).toHaveBeenCalledWith('npm_execpath');
+    });
+  });
+});

test/util/secureTempFiles.test.ts

@@ -0,0 +1,61 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+import {
+  createSecureTempDirectory,
+  removeSecureTempDirectory,
+  writeSecureTempFile,
+} from '../../src/util/secureTempFiles';
+
+describe('secure temporary files', () => {
+  let tempDirectory: string | undefined;
+
+  afterEach(async () => {
+    if (tempDirectory) {
+      await removeSecureTempDirectory(tempDirectory);
+      tempDirectory = undefined;
+    }
+  });
+
+  it('creates private directories and exclusively writes restricted files', async () => {
+    tempDirectory = await createSecureTempDirectory('promptfoo-secure-temp-test-');
+    const filePath = await writeSecureTempFile(tempDirectory, 'payload.json', '{"ok":true}');
+
+    if (process.platform !== 'win32') {
+      expect((await fs.stat(tempDirectory)).mode & 0o777).toBe(0o700);
+      expect((await fs.stat(filePath)).mode & 0o777).toBe(0o600);
+    }
+
+    await expect(
+      writeSecureTempFile(tempDirectory, 'payload.json', 'replacement'),
+    ).rejects.toMatchObject({ code: 'EEXIST' });
+  });
+
+  it('rejects filenames that are not simple leaf names', async () => {
+    tempDirectory = await createSecureTempDirectory('promptfoo-secure-temp-test-');
+
+    for (const filename of ['../payload.json', path.resolve('payload.json'), '.', '..', '']) {
+      await expect(writeSecureTempFile(tempDirectory, filename, 'data')).rejects.toThrow(
+        'Secure temporary file names must be simple leaf names',
+      );
+    }
+  });
+
+  it('rejects directory prefixes that are not simple leaf names', async () => {
+    for (const prefix of ['../promptfoo-', path.resolve('promptfoo-'), '.', '..', '']) {
+      await expect(createSecureTempDirectory(prefix)).rejects.toThrow(
+        'Secure temporary directory prefixes must be simple leaf names',
+      );
+    }
+  });
+
+  it('removes the temporary directory recursively', async () => {
+    tempDirectory = await createSecureTempDirectory('promptfoo-secure-temp-test-');
+    await writeSecureTempFile(tempDirectory, 'payload.json', 'data');
+
+    await removeSecureTempDirectory(tempDirectory);
+    await expect(fs.access(tempDirectory)).rejects.toMatchObject({ code: 'ENOENT' });
+    tempDirectory = undefined;
+  });
+});

test/util/server.test.ts

@@ -0,0 +1,341 @@
+import opener from 'opener';
+import { beforeEach, describe, expect, it, type MockedFunction, vi } from 'vitest';
+import { getDefaultPort, VERSION } from '../../src/constants';
+import logger from '../../src/logger';
+import * as remoteGeneration from '../../src/redteam/remoteGeneration';
+import * as readlineUtils from '../../src/util/readline';
+// Import the module under test after mocks are set up
+import {
+  __clearFeatureCache,
+  BrowserBehavior,
+  checkServerFeatureSupport,
+  checkServerRunning,
+  openBrowser,
+} from '../../src/util/server';
+
+// Mock opener
+vi.mock('opener', () => ({ default: vi.fn() }));
+
+// Mock logger
+vi.mock('../../src/logger', () => ({
+  default: {
+    info: vi.fn(),
+    debug: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+// Mock the readline utilities
+vi.mock('../../src/util/readline', () => ({
+  promptYesNo: vi.fn(),
+  promptUser: vi.fn(),
+  createReadlineInterface: vi.fn(),
+}));
+
+// Mock fetchWithProxy
+vi.mock('../../src/util/fetch/index', () => ({
+  fetchWithProxy: vi.fn(),
+}));
+
+// Mock remoteGeneration
+vi.mock('../../src/redteam/remoteGeneration', () => ({
+  getRemoteVersionUrl: vi.fn(),
+}));
+
+// Import the mocked fetchWithProxy for use in tests
+import * as fetchModule from '../../src/util/fetch/index';
+
+const mockFetchWithProxy = fetchModule.fetchWithProxy as MockedFunction<
+  typeof fetchModule.fetchWithProxy
+>;
+
+describe('Server Utilities', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('checkServerRunning', () => {
+    it('should return true when server is running with matching version', async () => {
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ status: 'OK', version: VERSION }),
+      } as Response);
+
+      const result = await checkServerRunning();
+
+      expect(mockFetchWithProxy).toHaveBeenCalledWith(
+        `http://localhost:${getDefaultPort()}/health`,
+        {
+          headers: {
+            'x-promptfoo-silent': 'true',
+          },
+        },
+      );
+      expect(result).toBe(true);
+    });
+
+    it('should return false when server status is not OK', async () => {
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ status: 'ERROR', version: VERSION }),
+      } as Response);
+
+      const result = await checkServerRunning();
+
+      expect(result).toBe(false);
+    });
+
+    it('should return false when server version does not match', async () => {
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ status: 'OK', version: 'wrong-version' }),
+      } as Response);
+
+      const result = await checkServerRunning();
+
+      expect(result).toBe(false);
+    });
+
+    it('should return false when fetch throws an error', async () => {
+      mockFetchWithProxy.mockRejectedValueOnce(new Error('Connection refused'));
+
+      const result = await checkServerRunning();
+
+      expect(result).toBe(false);
+      expect(logger.debug).toHaveBeenCalledWith(
+        expect.stringContaining('No existing server found'),
+      );
+    });
+
+    it('should use custom port when provided', async () => {
+      const customPort = 4000;
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ status: 'OK', version: VERSION }),
+      } as Response);
+
+      await checkServerRunning(customPort);
+
+      expect(mockFetchWithProxy).toHaveBeenCalledWith(`http://localhost:${customPort}/health`, {
+        headers: {
+          'x-promptfoo-silent': 'true',
+        },
+      });
+    });
+  });
+
+  describe('openBrowser', () => {
+    it('should open browser with default URL when BrowserBehavior.OPEN', async () => {
+      await openBrowser(BrowserBehavior.OPEN);
+
+      expect(opener).toHaveBeenCalledWith(`http://localhost:${getDefaultPort()}`);
+      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('Press Ctrl+C'));
+    });
+
+    it('should open browser with report URL when BrowserBehavior.OPEN_TO_REPORT', async () => {
+      await openBrowser(BrowserBehavior.OPEN_TO_REPORT);
+
+      expect(opener).toHaveBeenCalledWith(`http://localhost:${getDefaultPort()}/report`);
+    });
+
+    it('should open browser with redteam setup URL when BrowserBehavior.OPEN_TO_REDTEAM_CREATE', async () => {
+      await openBrowser(BrowserBehavior.OPEN_TO_REDTEAM_CREATE);
+
+      expect(opener).toHaveBeenCalledWith(`http://localhost:${getDefaultPort()}/redteam/setup`);
+    });
+
+    it('should open browser with eval setup URL when BrowserBehavior.OPEN_TO_EVAL_SETUP', async () => {
+      await openBrowser(BrowserBehavior.OPEN_TO_EVAL_SETUP);
+
+      expect(opener).toHaveBeenCalledWith(`http://localhost:${getDefaultPort()}/setup`);
+    });
+
+    it('should not open browser when BrowserBehavior.SKIP', async () => {
+      await openBrowser(BrowserBehavior.SKIP);
+
+      expect(opener).not.toHaveBeenCalled();
+      expect(logger.info).not.toHaveBeenCalled();
+    });
+
+    it('should handle opener errors gracefully', async () => {
+      vi.mocked(opener).mockImplementationOnce(() => {
+        throw new Error('Failed to open browser');
+      });
+
+      await openBrowser(BrowserBehavior.OPEN);
+
+      expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('Failed to open browser'));
+    });
+
+    it('should ask user before opening browser when BrowserBehavior.ASK', async () => {
+      // Mock promptYesNo to return true
+      vi.mocked(readlineUtils.promptYesNo).mockResolvedValueOnce(true);
+
+      await openBrowser(BrowserBehavior.ASK);
+
+      expect(readlineUtils.promptYesNo).toHaveBeenCalledWith('Open URL in browser?', false);
+      expect(opener).toHaveBeenCalledWith(`http://localhost:${getDefaultPort()}`);
+    });
+
+    it('should not open browser when user answers no to ASK prompt', async () => {
+      // Mock promptYesNo to return false
+      vi.mocked(readlineUtils.promptYesNo).mockResolvedValueOnce(false);
+
+      await openBrowser(BrowserBehavior.ASK);
+
+      expect(readlineUtils.promptYesNo).toHaveBeenCalledWith('Open URL in browser?', false);
+      expect(opener).not.toHaveBeenCalled();
+    });
+
+    it('should use custom port when provided', async () => {
+      const customPort = 5000;
+      await openBrowser(BrowserBehavior.OPEN, customPort);
+
+      expect(opener).toHaveBeenCalledWith(`http://localhost:${customPort}`);
+    });
+  });
+
+  describe('checkServerFeatureSupport', () => {
+    const featureName = 'test-feature';
+
+    beforeEach(() => {
+      // Clear the feature cache before each test to ensure isolation
+      __clearFeatureCache();
+      vi.clearAllMocks();
+      // Setup default mock for getRemoteVersionUrl to return a valid URL
+      vi.mocked(remoteGeneration.getRemoteVersionUrl).mockReturnValue(
+        'https://api.promptfoo.app/version',
+      );
+    });
+
+    it('should return true when server buildDate is after required date', async () => {
+      const requiredDate = '2024-01-01T00:00:00Z';
+      const serverBuildDate = '2024-06-15T10:30:00Z';
+
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ buildDate: serverBuildDate, version: '1.0.0' }),
+      } as Response);
+
+      const result = await checkServerFeatureSupport(featureName, requiredDate);
+
+      expect(mockFetchWithProxy).toHaveBeenCalledWith('https://api.promptfoo.app/version', {
+        method: 'GET',
+        headers: { 'Content-Type': 'application/json' },
+      });
+      expect(result).toBe(true);
+      expect(logger.debug).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `${featureName}: buildDate=${serverBuildDate}, required=${requiredDate}, supported=true`,
+        ),
+      );
+    });
+
+    it('should return false when server buildDate is before required date', async () => {
+      const requiredDate = '2024-06-01T00:00:00Z';
+      const serverBuildDate = '2024-01-15T10:30:00Z';
+
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ buildDate: serverBuildDate, version: '1.0.0' }),
+      } as Response);
+
+      const result = await checkServerFeatureSupport(featureName, requiredDate);
+
+      expect(result).toBe(false);
+      expect(logger.debug).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `${featureName}: buildDate=${serverBuildDate}, required=${requiredDate}, supported=false`,
+        ),
+      );
+    });
+
+    it('should return false when no version info available', async () => {
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({}),
+      } as Response);
+
+      const result = await checkServerFeatureSupport(featureName, '2024-01-01T00:00:00Z');
+
+      expect(result).toBe(false);
+      expect(logger.debug).toHaveBeenCalledWith(
+        expect.stringContaining(`${featureName}: no version info, assuming not supported`),
+      );
+    });
+
+    it('should return true when no remote URL is available (local server assumption)', async () => {
+      // Mock getRemoteVersionUrl to return null for this specific test
+      vi.mocked(remoteGeneration.getRemoteVersionUrl).mockReturnValueOnce(null);
+
+      const result = await checkServerFeatureSupport(featureName, '2024-01-01T00:00:00Z');
+
+      expect(result).toBe(true);
+      expect(mockFetchWithProxy).not.toHaveBeenCalled();
+      expect(logger.debug).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `No remote URL available for ${featureName}, assuming local server supports it`,
+        ),
+      );
+    });
+
+    it('should return false when fetchWithProxy throws an error', async () => {
+      mockFetchWithProxy.mockRejectedValueOnce(new Error('Network error'));
+
+      const result = await checkServerFeatureSupport(featureName, '2024-01-01T00:00:00Z');
+
+      expect(result).toBe(false);
+      expect(logger.debug).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `Version check failed for ${featureName}, assuming not supported: Error: Network error`,
+        ),
+      );
+    });
+
+    it('should cache results to avoid repeated API calls', async () => {
+      const requiredDate = '2024-01-01T00:00:00Z';
+      const serverBuildDate = '2024-06-15T10:30:00Z';
+
+      mockFetchWithProxy.mockResolvedValue({
+        json: async () => ({ buildDate: serverBuildDate, version: '1.0.0' }),
+      } as Response);
+
+      // First call
+      const result1 = await checkServerFeatureSupport(featureName, requiredDate);
+      // Second call with same parameters
+      const result2 = await checkServerFeatureSupport(featureName, requiredDate);
+
+      expect(result1).toBe(true);
+      expect(result2).toBe(true);
+      // fetchWithProxy should only be called once due to caching
+      expect(mockFetchWithProxy).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle different feature names with separate cache entries', async () => {
+      const feature1 = 'feature-one';
+      const feature2 = 'feature-two';
+      const requiredDate = '2024-01-01T00:00:00Z';
+
+      mockFetchWithProxy
+        .mockResolvedValueOnce({
+          json: async () => ({ buildDate: '2024-06-15T10:30:00Z', version: '1.0.0' }),
+        } as Response)
+        .mockResolvedValueOnce({
+          json: async () => ({ buildDate: '2023-12-15T10:30:00Z', version: '1.0.0' }),
+        } as Response);
+
+      const result1 = await checkServerFeatureSupport(feature1, requiredDate);
+      const result2 = await checkServerFeatureSupport(feature2, requiredDate);
+
+      expect(result1).toBe(true);
+      expect(result2).toBe(false);
+      expect(mockFetchWithProxy).toHaveBeenCalledTimes(2);
+    });
+
+    it('should handle timezone differences correctly', async () => {
+      const requiredDate = '2024-06-01T00:00:00Z'; // UTC
+      const serverBuildDate = '2024-06-01T08:00:00+08:00'; // Same moment in different timezone
+
+      mockFetchWithProxy.mockResolvedValueOnce({
+        json: async () => ({ buildDate: serverBuildDate, version: '1.0.0' }),
+      } as Response);
+
+      const result = await checkServerFeatureSupport(featureName, requiredDate);
+
+      expect(result).toBe(true);
+    });
+  });
+});

test/util/sharing.test.ts

@@ -0,0 +1,92 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { shouldShareResults } from '../../src/util/sharing';
+
+vi.mock('../../src/envars', () => ({
+  getEnvBool: vi.fn().mockReturnValue(false),
+}));
+
+vi.mock('../../src/globalConfig/cloud', () => ({
+  cloudConfig: {
+    isEnabled: vi.fn().mockReturnValue(false),
+    getSharing: vi.fn().mockReturnValue(undefined),
+  },
+}));
+
+import { getEnvBool } from '../../src/envars';
+import { cloudConfig } from '../../src/globalConfig/cloud';
+
+describe('shouldShareResults', () => {
+  beforeEach(() => {
+    vi.mocked(getEnvBool).mockReturnValue(false);
+    vi.mocked(cloudConfig.isEnabled).mockReturnValue(false);
+    vi.mocked(cloudConfig.getSharing).mockReturnValue(undefined);
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  describe('explicit disable', () => {
+    it('should return false when cliNoShare is true', () => {
+      expect(shouldShareResults({ cliNoShare: true })).toBe(false);
+    });
+
+    it('should return false when cliShare is false', () => {
+      expect(shouldShareResults({ cliShare: false })).toBe(false);
+    });
+
+    it('should return false when PROMPTFOO_DISABLE_SHARING env var is set', () => {
+      vi.mocked(getEnvBool).mockReturnValue(true);
+      expect(shouldShareResults({})).toBe(false);
+    });
+  });
+
+  describe('explicit enable', () => {
+    it('should return true when cliShare is true', () => {
+      expect(shouldShareResults({ cliShare: true })).toBe(true);
+    });
+  });
+
+  describe('config file settings', () => {
+    it('should return true when configShare is truthy', () => {
+      expect(shouldShareResults({ configShare: true })).toBe(true);
+    });
+
+    it('should return false when configShare is false', () => {
+      expect(shouldShareResults({ configShare: false })).toBe(false);
+    });
+
+    it('should return true when configSharing is truthy', () => {
+      expect(shouldShareResults({ configSharing: true })).toBe(true);
+    });
+
+    it('should return false when configSharing is false', () => {
+      expect(shouldShareResults({ configSharing: false })).toBe(false);
+    });
+  });
+
+  describe('default behavior', () => {
+    it('should return false when cloud is not enabled', () => {
+      vi.mocked(cloudConfig.isEnabled).mockReturnValue(false);
+      expect(shouldShareResults({})).toBe(false);
+    });
+
+    it('should return true when cloud is enabled but sharing is undefined (pre-migration backward compat)', () => {
+      vi.mocked(cloudConfig.isEnabled).mockReturnValue(true);
+      vi.mocked(cloudConfig.getSharing).mockReturnValue(undefined);
+      expect(shouldShareResults({})).toBe(true);
+    });
+
+    it('should return false when cloud is enabled but sharing is false', () => {
+      vi.mocked(cloudConfig.isEnabled).mockReturnValue(true);
+      vi.mocked(cloudConfig.getSharing).mockReturnValue(false);
+      expect(shouldShareResults({})).toBe(false);
+    });
+
+    it('should return true when cloud is enabled and sharing is true', () => {
+      vi.mocked(cloudConfig.isEnabled).mockReturnValue(true);
+      vi.mocked(cloudConfig.getSharing).mockReturnValue(true);
+      expect(shouldShareResults({})).toBe(true);
+    });
+  });
+});

test/util/templates.test.ts

@@ -0,0 +1,497 @@
+import nunjucks from 'nunjucks';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import cliState from '../../src/cliState';
+import {
+  analyzeTemplateReference,
+  extractVariablesFromTemplate,
+  extractVariablesFromTemplates,
+  getNunjucksEngine,
+  templateReferencesVariable,
+} from '../../src/util/templates';
+import { mockProcessEnv } from './utils';
+
+describe('extractVariablesFromTemplate', () => {
+  it('should extract simple variables', () => {
+    const template = 'Hello {{ name }}, welcome to {{ place }}!';
+    expect(extractVariablesFromTemplate(template)).toEqual(['name', 'place']);
+  });
+
+  it('should extract variables without spaces', () => {
+    const template = 'Hello {{name}}, welcome to {{place}}!';
+    expect(extractVariablesFromTemplate(template)).toEqual(['name', 'place']);
+  });
+
+  it('should extract variables with dot notation', () => {
+    const template = 'Hello {{ user.name }}, your score is {{ game.score }}!';
+    expect(extractVariablesFromTemplate(template)).toEqual(['user.name', 'game.score']);
+  });
+
+  it('should extract variables with underscores', () => {
+    const template = 'Your order {{ order_id }} will arrive on {{ delivery_date }}.';
+    expect(extractVariablesFromTemplate(template)).toEqual(['order_id', 'delivery_date']);
+  });
+
+  it('should extract variables with numbers', () => {
+    const template = 'Player1: {{ player1 }}, Player2: {{ player2 }}';
+    expect(extractVariablesFromTemplate(template)).toEqual(['player1', 'player2']);
+  });
+
+  it('should extract variables used in filters', () => {
+    const template = '{{ name | capitalize }} - {{ date | date("yyyy-MM-dd") }}';
+    expect(extractVariablesFromTemplate(template)).toEqual(['name', 'date']);
+  });
+
+  it('should extract variables used in complex expressions', () => {
+    const template = '{% if user.age > 18 %}Welcome, {{ user.name }}!{% endif %}';
+    expect(extractVariablesFromTemplate(template)).toEqual(['user.age', 'user.name']);
+  });
+
+  it('should extract variables from for loops', () => {
+    const template = '{% for item in items %}{{ item.name }}{% endfor %}';
+    expect(extractVariablesFromTemplate(template)).toEqual(['item.name', 'items']);
+  });
+
+  it('should extract variables with multiple occurrences', () => {
+    const template = '{{ name }} {{ age }} {{ name }}';
+    expect(extractVariablesFromTemplate(template)).toEqual(['name', 'age']);
+  });
+
+  it('should not extract variables from comments', () => {
+    const template = '{# This is a comment with {{ variable }} #}{{ actual_variable }}';
+    expect(extractVariablesFromTemplate(template)).toEqual(['actual_variable']);
+  });
+
+  it('should handle empty templates', () => {
+    const template = '';
+    expect(extractVariablesFromTemplate(template)).toEqual([]);
+  });
+
+  it('should handle templates without variables', () => {
+    const template = 'This is a static template without variables.';
+    expect(extractVariablesFromTemplate(template)).toEqual([]);
+  });
+});
+
+describe('extractVariablesFromTemplates', () => {
+  it('should extract variables from multiple templates', () => {
+    const templates = [
+      'Hello {{ name }}, welcome to {{ place }}!',
+      '{{ user.age }} years old',
+      '{% for item in items %}{{ item.name }}{% endfor %}',
+    ];
+
+    const result = extractVariablesFromTemplates(templates);
+
+    expect(result).toEqual(['name', 'place', 'user.age', 'item.name', 'items']);
+  });
+
+  it('should handle empty array of templates', () => {
+    const templates: string[] = [];
+
+    const result = extractVariablesFromTemplates(templates);
+
+    expect(result).toEqual([]);
+  });
+
+  it('should deduplicate variables across templates', () => {
+    const templates = ['Hello {{ name }}!', 'Welcome, {{ name }}!', '{{ age }} years old'];
+
+    const result = extractVariablesFromTemplates(templates);
+
+    expect(result).toEqual(['name', 'age']);
+  });
+});
+
+describe('templateReferencesVariable', () => {
+  it.each<[string, string]>([
+    ['basic symbol read', '{{ _conversation[0].output }}'],
+    ['filter input', '{{ _conversation | length }}'],
+    ['chained filters', '{{ _conversation | first | length }}'],
+    ['if condition', '{% if _conversation %}yes{% endif %}'],
+    ['elif condition', '{% if false %}no{% elif _conversation %}yes{% endif %}'],
+    ['ternary true branch', '{{ "yes" if _conversation else "no" }}'],
+    ['ternary condition', '{{ first if _conversation else second }}'],
+    ['ternary else branch', '{{ a if x else _conversation }}'],
+    ['logical expression', '{{ _conversation and foo }}'],
+    ['not expression', '{{ not _conversation }}'],
+    ['comparison', '{{ _conversation == [] }}'],
+    ['less-than', '{{ x < _conversation }}'],
+    ['for loop iterable', '{% for turn in _conversation %}{{ turn.output }}{% endfor %}'],
+    ['function argument', '{{ summarize(_conversation) }}'],
+    ['keyword argument value', '{{ fn(arg=_conversation) }}'],
+    ['filter argument', '{{ input | default(_conversation) }}'],
+    ['is test with call arg', '{{ foo is divisibleby(_conversation) }}'],
+    ['is test function argument value', '{{ foo is _conversation(_conversation) }}'],
+    ['bracket index lookup', '{{ obj[_conversation] }}'],
+    ['array literal member', '{{ [_conversation, other] }}'],
+    ['group expression', '{{ (_conversation) }}'],
+    ['negated expression', '{{ -_conversation }}'],
+    ['concat', '{{ "a" ~ _conversation ~ "b" }}'],
+    ['dict literal value', '{{ {"history": _conversation}["history"][0].output }}'],
+    ['set value binding', '{% set history = _conversation %}{{ history }}'],
+    ['set block binding', '{% set history %}{{ _conversation }}{% endset %}{{ history }}'],
+    ['macro body free variable', '{% macro render() %}{{ _conversation }}{% endmacro %}'],
+    ['macro default value', '{% macro render(x=_conversation) %}{{ x }}{% endmacro %}'],
+    [
+      'macro default value with body ref',
+      '{% macro render(x=_conversation) %}{{ _conversation }}{% endmacro %}',
+    ],
+    [
+      'macro self-referential default value',
+      '{% macro render(_conversation=_conversation) %}{{ _conversation[0].output }}{% endmacro %}{{ render() }}',
+    ],
+    ['include expression', '{% include _conversation %}'],
+    ['extends expression', '{% extends _conversation %}{% block b %}{% endblock %}'],
+    ['from-import template expression', '{% from template_name import foo %}{{ _conversation }}'],
+    ['block body reference', '{% block body %}{{ _conversation }}{% endblock %}'],
+    [
+      'call block body reference',
+      '{% macro m() %}{{ caller() }}{% endmacro %}{% call m() %}{{ _conversation }}{% endcall %}',
+    ],
+    [
+      'call block body reference with unrelated caller arg',
+      '{% macro m() %}{{ caller("local") }}{% endmacro %}{% call(x) m() %}{{ _conversation }}{% endcall %}',
+    ],
+    [
+      'call block caller default value',
+      '{% macro m() %}{{ caller() }}{% endmacro %}{% call(x=_conversation) m() %}{{ x[0].output }}{% endcall %}',
+    ],
+    [
+      'call block caller self-referential default value',
+      '{% macro m() %}{{ caller() }}{% endmacro %}{% call(_conversation=_conversation) m() %}{{ _conversation[0].output }}{% endcall %}',
+    ],
+    ['asyncEach iterable', '{% asyncEach turn in _conversation %}{{ turn.output }}{% endeach %}'],
+    ['asyncAll iterable', '{% asyncAll turn in _conversation %}{{ turn.output }}{% endall %}'],
+  ])('detects a real reference in %s', (_label, template) => {
+    expect(templateReferencesVariable(template, '_conversation')).toBe(true);
+  });
+
+  it.each<[string, string]>([
+    ['substring in plain text', 'Summarize the pre_conversation_context for {{ question }}'],
+    ['substring in symbol name', '{{ pre_conversation_context }}'],
+    ['string literal', '{{ "_conversation" }}'],
+    ['comment mention', '{# _conversation #}{{ question }}'],
+    ['property of another object', '{{ foo._conversation }}'],
+    ['bracket lookup on another object', '{{ foo["_conversation"] }}'],
+    ['dict literal key', '{{ {_conversation: input} }}'],
+    ['filter name', '{{ input | _conversation }}'],
+    ['is test name', '{{ input is _conversation }}'],
+    ['is test function name', '{{ input is _conversation(foo) }}'],
+    ['raw block', '{% raw %}{{ _conversation }}{% endraw %}'],
+    ['for-loop target shadow', '{% for _conversation in items %}{{ _conversation }}{% endfor %}'],
+    ['macro argument shadow', '{% macro render(_conversation) %}{{ _conversation }}{% endmacro %}'],
+    ['set value shadow', '{% set _conversation = [] %}{{ _conversation }}'],
+    ['set block shadow', '{% set _conversation %}local{% endset %}{{ _conversation }}'],
+    [
+      'set inside loop body',
+      '{% for i in range(3) %}{% set _conversation = [] %}{{ _conversation }}{% endfor %}',
+    ],
+    ['import target shadow', '{% import "x.njk" as _conversation %}{{ _conversation }}'],
+    ['from-import alias binding', '{% from "x.njk" import foo as _conversation %}{{ question }}'],
+    ['block name label', '{% block _conversation %}hi{% endblock %}'],
+    [
+      'for-loop tuple target shadow',
+      '{% for k, _conversation in items %}{{ _conversation }}{% endfor %}',
+    ],
+    [
+      'nested for target shadow',
+      '{% for _conversation in outer %}{% for x in _conversation %}{{ _conversation }}{% endfor %}{% endfor %}',
+    ],
+    ['set tuple target shadow', '{% set a, _conversation = pair %}{{ _conversation }}'],
+    [
+      'macro keyword-default param name shadow',
+      '{% macro render(_conversation=something) %}{{ _conversation }}{% endmacro %}',
+    ],
+    [
+      'macro keyword default left-to-right shadow',
+      '{% macro render(_conversation=[], history=_conversation) %}{{ history | length }}{% endmacro %}{{ render() }}',
+    ],
+    [
+      'macro positional param used by keyword default',
+      '{% macro render(_conversation, history=_conversation) %}{{ history }}{% endmacro %}',
+    ],
+    [
+      'call block caller argument shadow',
+      '{% macro m() %}{{ caller("local") }}{% endmacro %}{% call(_conversation) m() %}{{ _conversation }}{% endcall %}',
+    ],
+    [
+      'call block caller keyword default left-to-right shadow',
+      '{% macro m() %}{{ caller() }}{% endmacro %}{% call(_conversation=[], history=_conversation) m() %}{{ history | length }}{% endcall %}',
+    ],
+    [
+      'asyncEach target shadow',
+      '{% asyncEach _conversation in items %}{{ _conversation }}{% endeach %}',
+    ],
+    [
+      'asyncAll target shadow',
+      '{% asyncAll _conversation in items %}{{ _conversation }}{% endall %}',
+    ],
+  ])('ignores %s', (_label, template) => {
+    expect(templateReferencesVariable(template, '_conversation')).toBe(false);
+  });
+
+  it('handles from-import aliases without shadowing source names', () => {
+    expect(templateReferencesVariable('{% from "x.njk" import foo %}{{ foo }}', 'foo')).toBe(false);
+    expect(
+      templateReferencesVariable('{% from "x.njk" import foo as helper %}{{ helper }}', 'helper'),
+    ).toBe(false);
+    expect(
+      templateReferencesVariable('{% from "x.njk" import foo as helper %}{{ foo }}', 'foo'),
+    ).toBe(true);
+    expect(
+      templateReferencesVariable(
+        '{% from templateName import foo as helper %}{{ helper }}',
+        'templateName',
+      ),
+    ).toBe(true);
+  });
+
+  it('returns false for empty variable names', () => {
+    expect(templateReferencesVariable('{{ _conversation }}', '')).toBe(false);
+  });
+
+  it('returns false for templates that do not mention the variable at all', () => {
+    expect(templateReferencesVariable('{{ question }}', '_conversation')).toBe(false);
+    expect(templateReferencesVariable('', '_conversation')).toBe(false);
+  });
+
+  it('falls back to a conservative match when the template fails to parse', () => {
+    // Old substring-based code would force serial execution here; the new
+    // AST-based code must preserve that safety envelope when parsing fails
+    // rather than silently returning false.
+    expect(templateReferencesVariable('{{ _conversation', '_conversation')).toBe(true);
+    expect(templateReferencesVariable('{{ _conversation[0].output {% if %}', '_conversation')).toBe(
+      true,
+    );
+    expect(templateReferencesVariable('{{ _conversation }}{% endif %}', '_conversation')).toBe(
+      true,
+    );
+  });
+
+  it('does not throw on malformed templates', () => {
+    expect(() => templateReferencesVariable('{{ _conversation', '_conversation')).not.toThrow();
+    expect(() => templateReferencesVariable('{% for %}', '_conversation')).not.toThrow();
+  });
+});
+
+describe('analyzeTemplateReference', () => {
+  it('reports parsed=true when the template parses', () => {
+    expect(analyzeTemplateReference('{{ _conversation }}', '_conversation')).toEqual({
+      referenced: true,
+      parsed: true,
+    });
+    expect(analyzeTemplateReference('{{ question }}', '_conversation')).toEqual({
+      referenced: false,
+      parsed: true,
+    });
+  });
+
+  it('reports parsed=false and conservative referenced=true on parse failure', () => {
+    expect(analyzeTemplateReference('{{ _conversation', '_conversation')).toEqual({
+      referenced: true,
+      parsed: false,
+    });
+  });
+
+  it('reports parsed=true with referenced=false when the variable is not mentioned', () => {
+    // The textual fast path returns parsed=true because nothing was actually parsed
+    // — no parse failure to surface.
+    expect(analyzeTemplateReference('{{ question }}', '_conversation')).toEqual({
+      referenced: false,
+      parsed: true,
+    });
+  });
+});
+
+describe('getNunjucksEngine', () => {
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    vi.resetModules();
+    mockProcessEnv({ ...originalEnv }, { clear: true });
+  });
+
+  afterEach(() => {
+    mockProcessEnv(originalEnv, { clear: true });
+  });
+
+  it('should return a nunjucks environment by default', () => {
+    const engine = getNunjucksEngine();
+    expect(engine).toBeInstanceOf(nunjucks.Environment);
+    expect(engine.renderString('Hello {{ name }}', { name: 'World' })).toBe('Hello World');
+  });
+
+  it('should return a simple render function when PROMPTFOO_DISABLE_TEMPLATING is set', () => {
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    const engine = getNunjucksEngine();
+    expect(engine.renderString('Hello {{ name }}', { name: 'World' })).toBe('Hello {{ name }}');
+  });
+
+  it('should return a nunjucks environment when isGrader is true, regardless of PROMPTFOO_DISABLE_TEMPLATING', () => {
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    const engine = getNunjucksEngine({}, false, true);
+    expect(engine).toBeInstanceOf(nunjucks.Environment);
+    expect(engine.renderString('Hello {{ name }}', { name: 'Grader' })).toBe('Hello Grader');
+  });
+
+  it('should use nunjucks when isGrader is true, even if PROMPTFOO_DISABLE_TEMPLATING is set', () => {
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    const engine = getNunjucksEngine({}, false, true);
+    expect(engine).toBeInstanceOf(nunjucks.Environment);
+    expect(engine.renderString('Hello {{ name }}', { name: 'Grader' })).toBe('Hello Grader');
+  });
+
+  it('should add custom filters when provided', () => {
+    const customFilters = {
+      uppercase: (str: string) => str.toUpperCase(),
+      add: (a: number, b: number) => (a + b).toString(),
+    };
+    const engine = getNunjucksEngine(customFilters);
+    expect(engine.renderString('{{ "hello" | uppercase }}', {})).toBe('HELLO');
+    expect(engine.renderString('{{ 5 | add(3) }}', {})).toBe('8');
+  });
+
+  it('should add built-in load filter for JSON parsing', () => {
+    const engine = getNunjucksEngine();
+    const jsonString = '{"name": "test", "value": 42}';
+    const template = `{{ '${jsonString}' | load }}`;
+    const result = engine.renderString(template, {});
+    expect(result).toBe('[object Object]');
+
+    // Test that the filter actually parses JSON correctly
+    const jsonData = '{"key": "value"}';
+    const loadFilter = engine.getFilter('load');
+    expect(loadFilter).toBeDefined();
+    expect(loadFilter(jsonData)).toEqual({ key: 'value' });
+  });
+
+  it('should handle load filter with invalid JSON', () => {
+    const engine = getNunjucksEngine();
+    const loadFilter = engine.getFilter('load');
+    expect(loadFilter).toBeDefined();
+    expect(() => loadFilter('invalid json')).toThrow('Unexpected token');
+  });
+
+  it('should add environment variables as globals under "env"', () => {
+    mockProcessEnv({ TEST_VAR: 'test_value' });
+    const engine = getNunjucksEngine();
+    expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('test_value');
+  });
+
+  it('should throw an error when throwOnUndefined is true and a variable is undefined', () => {
+    const engine = getNunjucksEngine({}, true);
+    expect(() => {
+      engine.renderString('{{ undefined_var }}', {});
+    }).toThrow(/attempted to output null or undefined value/);
+  });
+
+  it('should not throw an error when throwOnUndefined is false and a variable is undefined', () => {
+    const engine = getNunjucksEngine({}, false);
+    expect(() => {
+      engine.renderString('{{ undefined_var }}', {});
+    }).not.toThrow();
+  });
+
+  it('should respect all parameters when provided', () => {
+    mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    const customFilters = {
+      double: (n: number) => (n * 2).toString(),
+    };
+    const engine = getNunjucksEngine(customFilters, true, true);
+    expect(engine).toBeInstanceOf(nunjucks.Environment);
+    expect(engine.renderString('{{ 5 | double }}', {})).toBe('10');
+    expect(() => {
+      engine.renderString('{{ undefined_var }}', {});
+    }).toThrow(/attempted to output null or undefined value/);
+  });
+
+  describe('environment variables as globals', () => {
+    it('should add environment variables as globals by default', () => {
+      mockProcessEnv({ TEST_VAR: 'test_value' });
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('test_value');
+    });
+
+    it('should merge cliState.config.env with process.env', () => {
+      const initialConfig = { ...cliState.config };
+
+      cliState.config = {
+        env: {
+          PROCESS_VAR: 'overridden_value',
+          CONFIG_VAR: 'config_value',
+        },
+      };
+      const engine = getNunjucksEngine();
+      const rendered = engine.renderString('{{ env.PROCESS_VAR }}', {});
+      expect(rendered).toBe('overridden_value');
+      expect(engine.renderString('{{ env.CONFIG_VAR }}', {})).toBe('config_value');
+      cliState.config = initialConfig;
+    });
+
+    it('should handle undefined cliState.config', () => {
+      mockProcessEnv({ TEST_VAR: 'test_value' });
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('test_value');
+    });
+
+    it('should handle undefined cliState.config.env', () => {
+      mockProcessEnv({ TEST_VAR: 'test_value' });
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('test_value');
+    });
+
+    it('should disable process.env but allow config env variables when PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS is true', () => {
+      mockProcessEnv({ TEST_VAR: 'test_value' });
+      mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS: 'true' });
+      const initialConfig = { ...cliState.config };
+      cliState.config = {
+        env: {
+          CONFIG_VAR: 'config_value',
+        },
+      };
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('');
+      expect(engine.renderString('{{ env.CONFIG_VAR }}', {})).toBe('config_value');
+      cliState.config = initialConfig;
+    });
+
+    it('should disable process.env but allow config env variables when PROMPTFOO_SELF_HOSTED is true', () => {
+      mockProcessEnv({ TEST_VAR: 'test_value' });
+      mockProcessEnv({ PROMPTFOO_SELF_HOSTED: 'true' });
+      const initialConfig = { ...cliState.config };
+      cliState.config = {
+        env: {
+          CONFIG_VAR: 'config_value',
+        },
+      };
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('');
+      expect(engine.renderString('{{ env.CONFIG_VAR }}', {})).toBe('config_value');
+      cliState.config = initialConfig;
+    });
+
+    it('should handle empty config env object', () => {
+      mockProcessEnv({ TEST_VAR: 'test_value' });
+      const initialConfig = { ...cliState.config };
+      cliState.config = {
+        env: {},
+      };
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.TEST_VAR }}', {})).toBe('test_value');
+      cliState.config = initialConfig;
+    });
+
+    it('should prioritize config env variables over process.env when both exist', () => {
+      mockProcessEnv({ SHARED_VAR: 'process_value' });
+      const initialConfig = { ...cliState.config };
+      cliState.config = {
+        env: {
+          SHARED_VAR: 'config_value',
+        },
+      };
+      const engine = getNunjucksEngine();
+      expect(engine.renderString('{{ env.SHARED_VAR }}', {})).toBe('config_value');
+      cliState.config = initialConfig;
+    });
+  });
+});

test/util/text.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+import { ellipsize } from '../../src/util/text';
+
+describe('ellipsize', () => {
+  it('should not modify string shorter than maxLen', () => {
+    const str = 'hello';
+    expect(ellipsize(str, 10)).toBe('hello');
+  });
+
+  it('should truncate string and add ellipsis when longer than maxLen', () => {
+    const str = 'hello world';
+    expect(ellipsize(str, 8)).toBe('hello...');
+  });
+
+  it('should handle string equal to maxLen', () => {
+    const str = 'hello';
+    expect(ellipsize(str, 5)).toBe('hello');
+  });
+
+  it('should handle very short maxLen', () => {
+    const str = 'hello';
+    expect(ellipsize(str, 4)).toBe('h...');
+  });
+
+  it('should return an empty string for negative maxLen', () => {
+    expect(ellipsize('hello', -1)).toBe('');
+    expect(ellipsize('hello', -5)).toBe('');
+  });
+
+  it('should handle empty string', () => {
+    expect(ellipsize('', 5)).toBe('');
+  });
+
+  it('should hard-truncate when maxLen is less than 3', () => {
+    // maxLen < 3 has no room for "..."; the result must still respect maxLen
+    // (previously slice(0, maxLen - 3) used a negative index and overflowed).
+    expect(ellipsize('hello', 2)).toBe('he');
+    expect(ellipsize('hello', 1)).toBe('h');
+    expect(ellipsize('hello', 0)).toBe('');
+    expect(ellipsize('hello', 3)).toBe('...');
+  });
+});

test/util/tokenUsage.test.ts

@@ -0,0 +1,252 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../src/logger';
+import { TokenUsageTracker } from '../../src/util/tokenUsage';
+
+import type { TokenUsage } from '../../src/types/shared';
+
+describe('TokenUsageTracker', () => {
+  it('redacts URL credentials in provider ids when logging tracked usage', () => {
+    // The debug message string is the only redaction layer for provider ids —
+    // logger sanitizes context objects, never message strings.
+    const debugSpy = vi.spyOn(logger, 'debug').mockImplementation(() => logger);
+    try {
+      TokenUsageTracker.getInstance().trackUsage(
+        'https://api.example.com/v1?api_key=sk-12345678901234567890 (HttpProvider)',
+        { total: 1 },
+      );
+      const messages = debugSpy.mock.calls.map((call) => String(call[0])).join('\n');
+      expect(messages).toContain('api_key=%5BREDACTED%5D');
+      expect(messages).toContain('(HttpProvider)');
+      expect(messages).not.toContain('sk-12345678901234567890');
+    } finally {
+      debugSpy.mockRestore();
+      TokenUsageTracker.getInstance().resetAllUsage();
+    }
+  });
+
+  let tracker: TokenUsageTracker;
+
+  beforeEach(() => {
+    tracker = TokenUsageTracker.getInstance();
+    tracker.resetAllUsage();
+  });
+
+  afterEach(() => {
+    tracker.cleanup();
+  });
+
+  it('should track token usage for a provider', () => {
+    const usage: TokenUsage = {
+      total: 100,
+      prompt: 50,
+      completion: 50,
+      cached: 10,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 20,
+        acceptedPrediction: 15,
+        rejectedPrediction: 5,
+      },
+      assertions: {
+        total: 30,
+        prompt: 10,
+        completion: 15,
+        cached: 5,
+      },
+    };
+
+    tracker.trackUsage('test-provider', usage);
+    const tracked = tracker.getProviderUsage('test-provider');
+
+    expect(tracked).toEqual({
+      ...usage,
+      completionDetails: {
+        reasoning: 20,
+        acceptedPrediction: 15,
+        rejectedPrediction: 5,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        ...usage.assertions,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+  });
+
+  it('should handle undefined token usage', () => {
+    tracker.trackUsage('test-provider', undefined);
+    expect(tracker.getProviderUsage('test-provider')).toEqual(
+      expect.objectContaining({ numRequests: 1 }),
+    );
+  });
+
+  it('should merge token usage for the same provider', () => {
+    const usage1: TokenUsage = {
+      total: 100,
+      prompt: 50,
+      completion: 50,
+      cached: 10,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 20,
+        acceptedPrediction: 15,
+        rejectedPrediction: 5,
+      },
+      assertions: {
+        total: 30,
+        prompt: 10,
+        completion: 15,
+        cached: 5,
+      },
+    };
+
+    const usage2: TokenUsage = {
+      total: 200,
+      prompt: 100,
+      completion: 100,
+      cached: 20,
+      numRequests: 2,
+      completionDetails: {
+        reasoning: 40,
+        acceptedPrediction: 30,
+        rejectedPrediction: 10,
+      },
+      assertions: {
+        total: 60,
+        prompt: 20,
+        completion: 30,
+        cached: 10,
+      },
+    };
+
+    tracker.trackUsage('test-provider', usage1);
+    tracker.trackUsage('test-provider', usage2);
+
+    const merged = tracker.getProviderUsage('test-provider');
+    expect(merged).toEqual({
+      total: 300,
+      prompt: 150,
+      completion: 150,
+      cached: 30,
+      numRequests: 3,
+      completionDetails: {
+        reasoning: 60,
+        acceptedPrediction: 45,
+        rejectedPrediction: 15,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 90,
+        prompt: 30,
+        completion: 45,
+        cached: 15,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+  });
+
+  it('should get provider IDs', () => {
+    tracker.trackUsage('provider1', { total: 100 });
+    tracker.trackUsage('provider2', { total: 200 });
+
+    expect(tracker.getProviderIds()).toEqual(['provider1', 'provider2']);
+  });
+
+  it('should get total usage across all providers', () => {
+    tracker.trackUsage('provider1', {
+      total: 100,
+      prompt: 50,
+      completion: 50,
+      cached: 10,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 20,
+        acceptedPrediction: 15,
+        rejectedPrediction: 5,
+      },
+      assertions: {
+        total: 30,
+        prompt: 10,
+        completion: 15,
+        cached: 5,
+      },
+    });
+
+    tracker.trackUsage('provider2', {
+      total: 200,
+      prompt: 100,
+      completion: 100,
+      cached: 20,
+      numRequests: 2,
+      completionDetails: {
+        reasoning: 40,
+        acceptedPrediction: 30,
+        rejectedPrediction: 10,
+      },
+      assertions: {
+        total: 60,
+        prompt: 20,
+        completion: 30,
+        cached: 10,
+      },
+    });
+
+    expect(tracker.getTotalUsage()).toEqual({
+      total: 300,
+      prompt: 150,
+      completion: 150,
+      cached: 30,
+      numRequests: 3,
+      completionDetails: {
+        reasoning: 60,
+        acceptedPrediction: 45,
+        rejectedPrediction: 15,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 90,
+        prompt: 30,
+        completion: 45,
+        cached: 15,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+  });
+
+  it('should reset provider usage', () => {
+    tracker.trackUsage('provider1', { total: 100 });
+    tracker.resetProviderUsage('provider1');
+    expect(tracker.getProviderUsage('provider1')).toBeUndefined();
+  });
+
+  it('should reset all usage', () => {
+    tracker.trackUsage('provider1', { total: 100 });
+    tracker.trackUsage('provider2', { total: 200 });
+    tracker.resetAllUsage();
+    expect(tracker.getProviderIds()).toHaveLength(0);
+  });
+});

test/util/tokenUsageCompat.test.ts

@@ -0,0 +1,673 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { TokenUsageTracker } from '../../src/util/tokenUsage';
+import {
+  aggregateUsageFromSpans,
+  extractUsageFromSpan,
+  getTokenUsage,
+  getTokenUsageByProvider,
+  getTokenUsageByTestIndex,
+  getTokenUsageFromEvaluation,
+  getTokenUsageFromTrace,
+} from '../../src/util/tokenUsageCompat';
+
+import type { SpanData } from '../../src/tracing/store';
+import type { TraceData } from '../../src/types/tracing';
+
+// Mock the TraceStore
+vi.mock('../../src/tracing/store', () => ({
+  getTraceStore: vi.fn(),
+}));
+
+// Mock the TokenUsageTracker
+vi.mock('../../src/util/tokenUsage', () => ({
+  TokenUsageTracker: {
+    getInstance: vi.fn(),
+  },
+}));
+
+import { getTraceStore } from '../../src/tracing/store';
+
+describe('tokenUsageCompat', () => {
+  type TraceStoreType = ReturnType<typeof getTraceStore>;
+  type TokenUsageTrackerInstance = ReturnType<typeof TokenUsageTracker.getInstance>;
+
+  const mockTraceStore = {
+    getSpans: vi.fn<TraceStoreType['getSpans']>(),
+    getTracesByEvaluation: vi.fn<TraceStoreType['getTracesByEvaluation']>(),
+  };
+
+  const mockTracker = {
+    getProviderUsage: vi.fn<TokenUsageTrackerInstance['getProviderUsage']>(),
+    getTotalUsage: vi.fn<TokenUsageTrackerInstance['getTotalUsage']>(),
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(getTraceStore).mockReturnValue(mockTraceStore as unknown as TraceStoreType);
+    vi.mocked(TokenUsageTracker.getInstance).mockReturnValue(
+      mockTracker as unknown as TokenUsageTrackerInstance,
+    );
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  describe('extractUsageFromSpan', () => {
+    it('should return undefined for spans without attributes', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+      };
+
+      expect(extractUsageFromSpan(span)).toBeUndefined();
+    });
+
+    it('should return undefined for spans without GenAI usage attributes', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.system': 'openai',
+          'gen_ai.request.model': 'gpt-4',
+        },
+      };
+
+      expect(extractUsageFromSpan(span)).toBeUndefined();
+    });
+
+    it('should extract input tokens', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.usage.input_tokens': 100,
+        },
+      };
+
+      const usage = extractUsageFromSpan(span);
+      expect(usage).toEqual({
+        numRequests: 1,
+        prompt: 100,
+      });
+    });
+
+    it('should extract output tokens', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.usage.output_tokens': 50,
+        },
+      };
+
+      const usage = extractUsageFromSpan(span);
+      expect(usage).toEqual({
+        numRequests: 1,
+        completion: 50,
+      });
+    });
+
+    it('should extract all standard usage attributes', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.usage.input_tokens': 100,
+          'gen_ai.usage.output_tokens': 50,
+          'gen_ai.usage.total_tokens': 150,
+          'gen_ai.usage.cached_tokens': 25,
+        },
+      };
+
+      const usage = extractUsageFromSpan(span);
+      expect(usage).toEqual({
+        numRequests: 1,
+        prompt: 100,
+        completion: 50,
+        total: 150,
+        cached: 25,
+      });
+    });
+
+    it('should extract completion details attributes', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.usage.input_tokens': 100,
+          'gen_ai.usage.output_tokens': 50,
+          'gen_ai.usage.total_tokens': 150,
+          'gen_ai.usage.reasoning_tokens': 20,
+          'gen_ai.usage.accepted_prediction_tokens': 10,
+          'gen_ai.usage.rejected_prediction_tokens': 5,
+        },
+      };
+
+      const usage = extractUsageFromSpan(span);
+      expect(usage).toEqual({
+        numRequests: 1,
+        prompt: 100,
+        completion: 50,
+        total: 150,
+        completionDetails: {
+          reasoning: 20,
+          acceptedPrediction: 10,
+          rejectedPrediction: 5,
+        },
+      });
+    });
+
+    it('should extract cache token completion details attributes', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.usage.input_tokens': 100,
+          'gen_ai.usage.output_tokens': 50,
+          'gen_ai.usage.total_tokens': 150,
+          'gen_ai.usage.cache_read_input_tokens': 200,
+          'gen_ai.usage.cache_creation_input_tokens': 30,
+        },
+      };
+
+      const usage = extractUsageFromSpan(span);
+      expect(usage).toEqual({
+        numRequests: 1,
+        prompt: 100,
+        completion: 50,
+        total: 150,
+        completionDetails: {
+          cacheReadInputTokens: 200,
+          cacheCreationInputTokens: 30,
+        },
+      });
+    });
+
+    it('should ignore non-numeric attribute values', () => {
+      const span: SpanData = {
+        spanId: 'span-1',
+        name: 'test',
+        startTime: 0,
+        attributes: {
+          'gen_ai.usage.input_tokens': 'not-a-number',
+          'gen_ai.usage.output_tokens': 50,
+        },
+      };
+
+      const usage = extractUsageFromSpan(span);
+      expect(usage).toEqual({
+        numRequests: 1,
+        completion: 50,
+      });
+    });
+  });
+
+  describe('aggregateUsageFromSpans', () => {
+    it('should return empty usage for empty spans array', () => {
+      const result = aggregateUsageFromSpans([]);
+      expect(result).toEqual({
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        total: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+        assertions: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+          cached: 0,
+          numRequests: 0,
+          completionDetails: {
+            reasoning: 0,
+            acceptedPrediction: 0,
+            rejectedPrediction: 0,
+            cacheReadInputTokens: 0,
+            cacheCreationInputTokens: 0,
+          },
+        },
+      });
+    });
+
+    it('should aggregate usage from multiple spans', () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'chat gpt-4',
+          startTime: 0,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat gpt-4',
+          startTime: 1000,
+          attributes: {
+            'gen_ai.usage.input_tokens': 200,
+            'gen_ai.usage.output_tokens': 75,
+            'gen_ai.usage.total_tokens': 275,
+          },
+        },
+      ];
+
+      const result = aggregateUsageFromSpans(spans);
+      expect(result.prompt).toBe(300);
+      expect(result.completion).toBe(125);
+      expect(result.total).toBe(425);
+      expect(result.numRequests).toBe(2);
+    });
+
+    it('should skip spans without usage attributes', () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'internal-span',
+          startTime: 0,
+          attributes: {
+            'some.other.attribute': 'value',
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat gpt-4',
+          startTime: 1000,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+      ];
+
+      const result = aggregateUsageFromSpans(spans);
+      expect(result.prompt).toBe(100);
+      expect(result.completion).toBe(50);
+      expect(result.total).toBe(150);
+      expect(result.numRequests).toBe(1);
+    });
+
+    it('should aggregate completion details from multiple spans', () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'chat o1',
+          startTime: 0,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+            'gen_ai.usage.reasoning_tokens': 20,
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat o1',
+          startTime: 1000,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+            'gen_ai.usage.reasoning_tokens': 30,
+          },
+        },
+      ];
+
+      const result = aggregateUsageFromSpans(spans);
+      expect(result.completionDetails?.reasoning).toBe(50);
+    });
+  });
+
+  describe('getTokenUsageFromTrace', () => {
+    it('should fetch spans and aggregate usage', async () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'chat gpt-4',
+          startTime: 0,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+      ];
+
+      mockTraceStore.getSpans.mockResolvedValue(spans);
+
+      const result = await getTokenUsageFromTrace('trace-123');
+
+      expect(mockTraceStore.getSpans).toHaveBeenCalledWith('trace-123', {
+        sanitizeAttributes: false,
+        includeInternalSpans: true,
+      });
+      expect(result.prompt).toBe(100);
+      expect(result.completion).toBe(50);
+      expect(result.total).toBe(150);
+    });
+  });
+
+  describe('getTokenUsageFromEvaluation', () => {
+    it('should aggregate usage from all traces in evaluation', async () => {
+      const traces: TraceData[] = [
+        {
+          traceId: 'trace-1',
+          evaluationId: 'eval-456',
+          testCaseId: 'test-1',
+          spans: [
+            {
+              spanId: 'span-1',
+              name: 'chat gpt-4',
+              startTime: 0,
+              attributes: {
+                'gen_ai.usage.input_tokens': 100,
+                'gen_ai.usage.output_tokens': 50,
+                'gen_ai.usage.total_tokens': 150,
+              },
+            },
+          ],
+        },
+        {
+          traceId: 'trace-2',
+          evaluationId: 'eval-456',
+          testCaseId: 'test-2',
+          spans: [
+            {
+              spanId: 'span-2',
+              name: 'chat claude-3',
+              startTime: 0,
+              attributes: {
+                'gen_ai.usage.input_tokens': 200,
+                'gen_ai.usage.output_tokens': 100,
+                'gen_ai.usage.total_tokens': 300,
+              },
+            },
+          ],
+        },
+      ];
+
+      mockTraceStore.getTracesByEvaluation.mockResolvedValue(traces);
+
+      const result = await getTokenUsageFromEvaluation('eval-456');
+
+      expect(mockTraceStore.getTracesByEvaluation).toHaveBeenCalledWith('eval-456', {
+        sanitizeAttributes: false,
+      });
+      expect(result.prompt).toBe(300);
+      expect(result.completion).toBe(150);
+      expect(result.total).toBe(450);
+      expect(result.numRequests).toBe(2);
+    });
+  });
+
+  describe('getTokenUsage', () => {
+    it('should use OTEL data when traceId is provided', async () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'chat gpt-4',
+          startTime: 0,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+      ];
+
+      mockTraceStore.getSpans.mockResolvedValue(spans);
+
+      const result = await getTokenUsage({ traceId: 'trace-123' });
+
+      expect(mockTraceStore.getSpans).toHaveBeenCalled();
+      expect(result.prompt).toBe(100);
+    });
+
+    it('should use OTEL data when evalId is provided', async () => {
+      const traces: TraceData[] = [
+        {
+          traceId: 'trace-1',
+          evaluationId: 'eval-456',
+          testCaseId: 'test-1',
+          spans: [
+            {
+              spanId: 'span-1',
+              name: 'chat gpt-4',
+              startTime: 0,
+              attributes: {
+                'gen_ai.usage.input_tokens': 100,
+                'gen_ai.usage.output_tokens': 50,
+                'gen_ai.usage.total_tokens': 150,
+              },
+            },
+          ],
+        },
+      ];
+
+      mockTraceStore.getTracesByEvaluation.mockResolvedValue(traces);
+
+      const result = await getTokenUsage({ evalId: 'eval-456' });
+
+      expect(mockTraceStore.getTracesByEvaluation).toHaveBeenCalled();
+      expect(result.prompt).toBe(100);
+    });
+
+    it('should fall back to legacy tracker for providerId query', async () => {
+      mockTracker.getProviderUsage.mockReturnValue({
+        prompt: 500,
+        completion: 200,
+        total: 700,
+        numRequests: 5,
+      });
+
+      const result = await getTokenUsage({ providerId: 'openai:gpt-4' });
+
+      expect(mockTracker.getProviderUsage).toHaveBeenCalledWith('openai:gpt-4');
+      expect(result.prompt).toBe(500);
+    });
+
+    it('should fall back to legacy tracker getTotalUsage for empty query', async () => {
+      mockTracker.getTotalUsage.mockReturnValue({
+        prompt: 1000,
+        completion: 500,
+        total: 1500,
+        numRequests: 10,
+      });
+
+      const result = await getTokenUsage({});
+
+      expect(mockTracker.getTotalUsage).toHaveBeenCalled();
+      expect(result.total).toBe(1500);
+    });
+
+    it('should return empty usage when provider not found', async () => {
+      mockTracker.getProviderUsage.mockReturnValue(undefined);
+
+      const result = await getTokenUsage({ providerId: 'unknown-provider' });
+
+      expect(result.numRequests).toBe(0);
+    });
+  });
+
+  describe('getTokenUsageByProvider', () => {
+    it('should group usage by provider ID', async () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'chat gpt-4',
+          startTime: 0,
+          attributes: {
+            'promptfoo.provider.id': 'openai:gpt-4',
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat gpt-4',
+          startTime: 1000,
+          attributes: {
+            'promptfoo.provider.id': 'openai:gpt-4',
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-3',
+          name: 'chat claude-3',
+          startTime: 2000,
+          attributes: {
+            'promptfoo.provider.id': 'anthropic:claude-3-opus',
+            'gen_ai.usage.input_tokens': 200,
+            'gen_ai.usage.output_tokens': 100,
+            'gen_ai.usage.total_tokens': 300,
+          },
+        },
+      ];
+
+      mockTraceStore.getSpans.mockResolvedValue(spans);
+
+      const result = await getTokenUsageByProvider('trace-123');
+
+      expect(result.size).toBe(2);
+      expect(result.get('openai:gpt-4')?.total).toBe(300);
+      expect(result.get('openai:gpt-4')?.numRequests).toBe(2);
+      expect(result.get('anthropic:claude-3-opus')?.total).toBe(300);
+      expect(result.get('anthropic:claude-3-opus')?.numRequests).toBe(1);
+    });
+
+    it('should skip spans without provider ID', async () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'internal-span',
+          startTime: 0,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat gpt-4',
+          startTime: 1000,
+          attributes: {
+            'promptfoo.provider.id': 'openai:gpt-4',
+            'gen_ai.usage.input_tokens': 200,
+            'gen_ai.usage.output_tokens': 100,
+            'gen_ai.usage.total_tokens': 300,
+          },
+        },
+      ];
+
+      mockTraceStore.getSpans.mockResolvedValue(spans);
+
+      const result = await getTokenUsageByProvider('trace-123');
+
+      expect(result.size).toBe(1);
+      expect(result.get('openai:gpt-4')?.total).toBe(300);
+    });
+  });
+
+  describe('getTokenUsageByTestIndex', () => {
+    it('should group usage by test index', async () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'chat gpt-4',
+          startTime: 0,
+          attributes: {
+            'promptfoo.test.index': 0,
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat gpt-4',
+          startTime: 1000,
+          attributes: {
+            'promptfoo.test.index': 0,
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-3',
+          name: 'chat gpt-4',
+          startTime: 2000,
+          attributes: {
+            'promptfoo.test.index': 1,
+            'gen_ai.usage.input_tokens': 200,
+            'gen_ai.usage.output_tokens': 100,
+            'gen_ai.usage.total_tokens': 300,
+          },
+        },
+      ];
+
+      mockTraceStore.getSpans.mockResolvedValue(spans);
+
+      const result = await getTokenUsageByTestIndex('trace-123');
+
+      expect(result.size).toBe(2);
+      expect(result.get(0)?.total).toBe(300);
+      expect(result.get(0)?.numRequests).toBe(2);
+      expect(result.get(1)?.total).toBe(300);
+      expect(result.get(1)?.numRequests).toBe(1);
+    });
+
+    it('should skip spans without test index', async () => {
+      const spans: SpanData[] = [
+        {
+          spanId: 'span-1',
+          name: 'synthesis-span',
+          startTime: 0,
+          attributes: {
+            'gen_ai.usage.input_tokens': 100,
+            'gen_ai.usage.output_tokens': 50,
+            'gen_ai.usage.total_tokens': 150,
+          },
+        },
+        {
+          spanId: 'span-2',
+          name: 'chat gpt-4',
+          startTime: 1000,
+          attributes: {
+            'promptfoo.test.index': 0,
+            'gen_ai.usage.input_tokens': 200,
+            'gen_ai.usage.output_tokens': 100,
+            'gen_ai.usage.total_tokens': 300,
+          },
+        },
+      ];
+
+      mockTraceStore.getSpans.mockResolvedValue(spans);
+
+      const result = await getTokenUsageByTestIndex('trace-123');
+
+      expect(result.size).toBe(1);
+      expect(result.get(0)?.total).toBe(300);
+    });
+  });
+});

test/util/tokenUsageUtils.test.ts

@@ -0,0 +1,425 @@
+import { describe, expect, it } from 'vitest';
+import {
+  accumulateGradingRequest,
+  accumulateResponseTokenUsage,
+  accumulateTokenUsage,
+  createEmptyAssertions,
+  createEmptyTokenUsage,
+  normalizeTokenUsage,
+} from '../../src/util/tokenUsageUtils';
+
+import type { TokenUsage } from '../../src/types/shared';
+
+describe('tokenUsageUtils', () => {
+  describe('createEmptyTokenUsage', () => {
+    it('should create an empty token usage object with all fields initialized to zero', () => {
+      const result = createEmptyTokenUsage();
+
+      expect(result).toEqual({
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        total: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+        assertions: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+          cached: 0,
+          numRequests: 0,
+          completionDetails: {
+            reasoning: 0,
+            acceptedPrediction: 0,
+            rejectedPrediction: 0,
+            cacheReadInputTokens: 0,
+            cacheCreationInputTokens: 0,
+          },
+        },
+      });
+    });
+
+    it('should return Required<TokenUsage> type', () => {
+      const result = createEmptyTokenUsage();
+
+      // This test checks that all optional fields are actually present
+      expect(result.prompt).toBeDefined();
+      expect(result.completion).toBeDefined();
+      expect(result.cached).toBeDefined();
+      expect(result.total).toBeDefined();
+      expect(result.numRequests).toBeDefined();
+      expect(result.completionDetails).toBeDefined();
+      expect(result.assertions).toBeDefined();
+    });
+  });
+
+  describe('accumulateTokenUsage', () => {
+    it('should accumulate basic token fields', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+      const update = {
+        prompt: 10,
+        completion: 20,
+        cached: 5,
+        total: 30,
+      };
+
+      accumulateTokenUsage(target, update);
+
+      expect(target.prompt).toBe(10);
+      expect(target.completion).toBe(20);
+      expect(target.cached).toBe(5);
+      expect(target.total).toBe(30);
+    });
+
+    it('should handle undefined update gracefully', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+      const originalTarget = { ...target };
+
+      accumulateTokenUsage(target, undefined);
+
+      expect(target).toEqual(originalTarget);
+    });
+
+    it('should accumulate numRequests when provided', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+
+      accumulateTokenUsage(target, { numRequests: 3 });
+      expect(target.numRequests).toBe(3);
+
+      accumulateTokenUsage(target, { numRequests: 2 });
+      expect(target.numRequests).toBe(5);
+    });
+
+    it('should increment numRequests by 1 when incrementRequests is true and numRequests not provided', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+
+      accumulateTokenUsage(target, { total: 10 }, true);
+      expect(target.numRequests).toBe(1);
+
+      accumulateTokenUsage(target, { total: 5 }, true);
+      expect(target.numRequests).toBe(2);
+    });
+
+    it('should not increment numRequests when incrementRequests is false and numRequests not provided', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+
+      accumulateTokenUsage(target, { total: 10 }, false);
+      expect(target.numRequests).toBe(0);
+
+      accumulateTokenUsage(target, { total: 5 });
+      expect(target.numRequests).toBe(0);
+    });
+
+    it('should accumulate completion details', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+
+      accumulateTokenUsage(target, {
+        completionDetails: {
+          reasoning: 5,
+          acceptedPrediction: 3,
+          rejectedPrediction: 2,
+        },
+      });
+
+      expect(target.completionDetails).toMatchObject({
+        reasoning: 5,
+        acceptedPrediction: 3,
+        rejectedPrediction: 2,
+      });
+
+      accumulateTokenUsage(target, {
+        completionDetails: {
+          reasoning: 10,
+        },
+      });
+
+      expect(target.completionDetails).toMatchObject({
+        reasoning: 15,
+        acceptedPrediction: 3,
+        rejectedPrediction: 2,
+      });
+    });
+
+    it('should accumulate assertion tokens', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+
+      accumulateTokenUsage(target, {
+        assertions: {
+          total: 10,
+          prompt: 5,
+          completion: 5,
+          cached: 2,
+        },
+      });
+
+      expect(target.assertions?.total).toBe(10);
+      expect(target.assertions?.prompt).toBe(5);
+      expect(target.assertions?.completion).toBe(5);
+      expect(target.assertions?.cached).toBe(2);
+    });
+
+    it('should accumulate assertion completion details', () => {
+      const target: TokenUsage = createEmptyTokenUsage();
+
+      accumulateTokenUsage(target, {
+        assertions: {
+          completionDetails: {
+            reasoning: 5,
+            acceptedPrediction: 3,
+          },
+        },
+      });
+
+      expect(target.assertions?.completionDetails).toMatchObject({
+        reasoning: 5,
+        acceptedPrediction: 3,
+      });
+    });
+
+    it('should handle missing fields with undefined or 0', () => {
+      const target: TokenUsage = {
+        total: 10,
+        // Other fields undefined
+      };
+
+      accumulateTokenUsage(target, {
+        prompt: 5,
+        completion: 7,
+      });
+
+      expect(target.total).toBe(10);
+      expect(target.prompt).toBe(5);
+      expect(target.completion).toBe(7);
+      expect(target.cached).toBe(0); // addNumbers converts undefined to 0
+    });
+  });
+
+  describe('accumulateResponseTokenUsage', () => {
+    it('should accumulate token usage from response with tokenUsage', () => {
+      const target = createEmptyTokenUsage();
+      const response = {
+        tokenUsage: {
+          total: 100,
+          prompt: 60,
+          completion: 40,
+          numRequests: 1,
+        },
+      };
+
+      accumulateResponseTokenUsage(target, response);
+
+      expect(target.total).toBe(100);
+      expect(target.prompt).toBe(60);
+      expect(target.completion).toBe(40);
+      expect(target.numRequests).toBe(1);
+    });
+
+    it('should increment numRequests when response exists but has no tokenUsage', () => {
+      const target = createEmptyTokenUsage();
+      const response = {};
+
+      accumulateResponseTokenUsage(target, response);
+
+      expect(target.numRequests).toBe(1);
+      expect(target.total).toBe(0);
+    });
+
+    it('should handle undefined response', () => {
+      const target = createEmptyTokenUsage();
+
+      accumulateResponseTokenUsage(target, undefined);
+
+      expect(target.numRequests).toBe(0);
+      expect(target.total).toBe(0);
+    });
+
+    it('should accumulate multiple responses', () => {
+      const target = createEmptyTokenUsage();
+
+      accumulateResponseTokenUsage(target, {
+        tokenUsage: { total: 50, prompt: 30, completion: 20, numRequests: 1 },
+      });
+      accumulateResponseTokenUsage(target, {
+        tokenUsage: { total: 30, prompt: 20, completion: 10, numRequests: 1 },
+      });
+
+      expect(target.total).toBe(80);
+      expect(target.prompt).toBe(50);
+      expect(target.completion).toBe(30);
+      expect(target.numRequests).toBe(2);
+    });
+
+    it('should not increment numRequests when countAsRequest is false', () => {
+      const target = createEmptyTokenUsage();
+
+      accumulateResponseTokenUsage(
+        target,
+        {
+          tokenUsage: { total: 50, prompt: 30, completion: 20, numRequests: 1 },
+        },
+        { countAsRequest: false },
+      );
+
+      expect(target.total).toBe(50);
+      expect(target.prompt).toBe(30);
+      expect(target.completion).toBe(20);
+      expect(target.numRequests).toBe(0);
+    });
+
+    it('should not increment numRequests from response-only entries when countAsRequest is false', () => {
+      const target = createEmptyTokenUsage();
+
+      accumulateResponseTokenUsage(target, {}, { countAsRequest: false });
+
+      expect(target.total).toBe(0);
+      expect(target.numRequests).toBe(0);
+    });
+  });
+
+  describe('accumulateGradingRequest', () => {
+    it('counts the request without token usage when the grader reports none', () => {
+      const assertions = createEmptyAssertions();
+      accumulateGradingRequest(assertions, undefined);
+
+      expect(assertions.numRequests).toBe(1);
+      expect(assertions.total).toBe(0);
+    });
+
+    it('counts the request and folds in reported assertion token usage', () => {
+      const assertions = createEmptyAssertions();
+      accumulateGradingRequest(assertions, { total: 9, prompt: 5, completion: 4, numRequests: 3 });
+
+      expect(assertions.numRequests).toBe(1);
+      expect(assertions.total).toBe(9);
+      expect(assertions.prompt).toBe(5);
+      expect(assertions.completion).toBe(4);
+    });
+  });
+
+  describe('normalizeTokenUsage', () => {
+    it('should return fully populated TokenUsage with defaults for undefined input', () => {
+      const result = normalizeTokenUsage(undefined);
+
+      expect(result).toEqual({
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+        assertions: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+          cached: 0,
+          numRequests: 0,
+          completionDetails: {
+            reasoning: 0,
+            acceptedPrediction: 0,
+            rejectedPrediction: 0,
+            cacheReadInputTokens: 0,
+            cacheCreationInputTokens: 0,
+          },
+        },
+      });
+    });
+
+    it('should preserve provided values', () => {
+      const result = normalizeTokenUsage({
+        total: 100,
+        prompt: 60,
+        completion: 40,
+      });
+
+      expect(result.total).toBe(100);
+      expect(result.prompt).toBe(60);
+      expect(result.completion).toBe(40);
+    });
+
+    it('should fill in missing fields with defaults', () => {
+      const result = normalizeTokenUsage({
+        total: 50,
+      });
+
+      expect(result.total).toBe(50);
+      expect(result.prompt).toBe(0);
+      expect(result.completion).toBe(0);
+      expect(result.cached).toBe(0);
+      expect(result.numRequests).toBe(0);
+    });
+
+    it('should preserve completionDetails if provided', () => {
+      const result = normalizeTokenUsage({
+        completionDetails: {
+          reasoning: 10,
+          acceptedPrediction: 5,
+          rejectedPrediction: 2,
+        },
+      });
+
+      expect(result.completionDetails).toEqual({
+        reasoning: 10,
+        acceptedPrediction: 5,
+        rejectedPrediction: 2,
+      });
+    });
+
+    it('should preserve assertions if provided', () => {
+      const result = normalizeTokenUsage({
+        assertions: {
+          total: 20,
+          prompt: 10,
+          completion: 10,
+        },
+      });
+
+      expect(result.assertions.total).toBe(20);
+      expect(result.assertions.prompt).toBe(10);
+      expect(result.assertions.completion).toBe(10);
+    });
+
+    it('should handle empty object', () => {
+      const result = normalizeTokenUsage({});
+
+      expect(result.total).toBe(0);
+      expect(result.prompt).toBe(0);
+      expect(result.completion).toBe(0);
+    });
+
+    it('should handle partial completionDetails', () => {
+      const result = normalizeTokenUsage({
+        completionDetails: {
+          reasoning: 5,
+        },
+      });
+
+      expect(result.completionDetails.reasoning).toBe(5);
+      // Other fields may be undefined or 0 depending on source
+    });
+
+    it('should return Required<TokenUsage> type', () => {
+      const result = normalizeTokenUsage({ total: 10 });
+
+      // TypeScript compile-time check: all fields should be non-optional
+      expect(result.total).toBeDefined();
+      expect(result.prompt).toBeDefined();
+      expect(result.completion).toBeDefined();
+      expect(result.cached).toBeDefined();
+      expect(result.numRequests).toBeDefined();
+      expect(result.completionDetails).toBeDefined();
+      expect(result.assertions).toBeDefined();
+    });
+  });
+});

test/util/transform.test.ts

@@ -0,0 +1,608 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { importModule } from '../../src/esm';
+import logger from '../../src/logger';
+import { runPython } from '../../src/python/pythonUtils';
+import {
+  FILE_TRANSFORM_LABEL,
+  getTransformErrorMessage,
+  getTransformLabel,
+  INLINE_FUNCTION_LABEL,
+  INLINE_STRING_LABEL,
+  TransformInputType,
+  transform,
+} from '../../src/util/transform';
+
+import type { TransformFunction } from '../../src/types/transform';
+
+vi.mock('../../src/esm', () => ({
+  importModule: vi.fn(),
+  getDirectory: vi.fn().mockReturnValue('/test/dir'),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    error: vi.fn(),
+    warn: vi.fn(),
+    info: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+vi.mock('../../src/python/pythonUtils', () => ({
+  runPython: vi.fn().mockImplementation(async (_filePath, _functionName, args) => {
+    const [output] = args;
+    return output.toUpperCase() + ' FROM PYTHON';
+  }),
+}));
+
+vi.mock('fs', () => ({
+  unlink: vi.fn(),
+}));
+
+vi.mock('glob', () => ({
+  globSync: vi.fn(),
+}));
+
+vi.mock('../../src/database', () => ({
+  getDb: vi.fn(),
+}));
+
+const mockedImportModule = vi.mocked(importModule);
+
+describe('util', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('transform', () => {
+    afterEach(() => {
+      vi.clearAllMocks();
+      vi.resetModules();
+    });
+
+    it('transforms output using a direct function', async () => {
+      const output = 'original output';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      const transformFunction = 'output.toUpperCase()';
+      const transformedOutput = await transform(transformFunction, output, context);
+      expect(transformedOutput).toBe('ORIGINAL OUTPUT');
+    });
+
+    it('transforms vars using a direct function', async () => {
+      const vars = { key: 'value' };
+      const context = { vars: {}, prompt: { id: '123' } };
+      const transformFunction = 'JSON.stringify(vars)';
+      const transformedOutput = await transform(
+        transformFunction,
+        vars,
+        context,
+        true,
+        TransformInputType.VARS,
+      );
+      expect(transformedOutput).toBe('{"key":"value"}');
+    });
+
+    it('transforms output using an imported function from a file', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      mockedImportModule.mockResolvedValueOnce((output: string) => output.toUpperCase());
+
+      const transformFunctionPath = 'file://transform.js';
+      const transformedOutput = await transform(transformFunctionPath, output, context);
+      expect(transformedOutput).toBe('HELLO');
+    });
+
+    it('transforms vars using a direct function from a file', async () => {
+      const vars = { key: 'value' };
+      const context = { vars: {}, prompt: {} };
+      mockedImportModule.mockResolvedValueOnce((vars: any) => ({
+        ...vars,
+        key: 'transformed',
+      }));
+      const transformFunctionPath = 'file://transform.js';
+      const transformedOutput = await transform(
+        transformFunctionPath,
+        vars,
+        context,
+        true,
+        TransformInputType.VARS,
+      );
+      expect(transformedOutput).toEqual({ key: 'transformed' });
+    });
+
+    it('throws error if transform function does not return a value', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const transformFunction = ''; // Empty function, returns undefined
+      await expect(transform(transformFunction, output, context)).rejects.toThrow(
+        'Transform function did not return a value',
+      );
+    });
+
+    it('throws error if file does not export a function', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      mockedImportModule.mockResolvedValueOnce('banana');
+      const transformFunctionPath = 'file://transform.js';
+      await expect(transform(transformFunctionPath, output, context)).rejects.toThrow(
+        'Transform transform.js must export a function, have a default export as a function, or export the specified function "undefined"',
+      );
+      expect(logger.error).toHaveBeenCalledWith(
+        'Error loading transform function from file',
+        expect.objectContaining({
+          transform: FILE_TRANSFORM_LABEL,
+        }),
+      );
+    });
+
+    it('transforms output using a Python file', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      const pythonFilePath = 'file://transform.py';
+
+      const transformedOutput = await transform(pythonFilePath, output, context);
+      expect(transformedOutput).toBe('HELLO FROM PYTHON');
+    });
+
+    it('throws error for unsupported file format', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const unsupportedFilePath = 'file://transform.txt';
+
+      await expect(transform(unsupportedFilePath, output, context)).rejects.toThrow(
+        'Unsupported transform file format: file://transform.txt',
+      );
+      expect(logger.error).toHaveBeenCalledWith(
+        'Error loading transform function from file',
+        expect.objectContaining({
+          message: 'Unsupported transform file format: file://transform.txt',
+          transform: FILE_TRANSFORM_LABEL,
+        }),
+      );
+    });
+
+    it('transforms output using a multi-line function', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      const multiLineFunction = `
+        const uppercased = output.toUpperCase();
+        return uppercased + ' WORLD';
+      `;
+
+      const transformedOutput = await transform(multiLineFunction, output, context);
+      expect(transformedOutput).toBe('HELLO WORLD');
+    });
+
+    it('transforms output using a default export function from a file', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      mockedImportModule.mockResolvedValueOnce(
+        (output: string) => output.toUpperCase() + ' DEFAULT',
+      );
+
+      const transformFunctionPath = 'file://transform.js';
+      const transformedOutput = await transform(transformFunctionPath, output, context);
+      expect(transformedOutput).toBe('HELLO DEFAULT');
+    });
+
+    it('transforms output using a named function from a JavaScript file', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      // When a function name is specified, importModule returns that specific function
+      mockedImportModule.mockResolvedValueOnce((output: string) => output.toUpperCase() + ' NAMED');
+
+      const transformFunctionPath = 'file://transform.js:namedFunction';
+      const transformedOutput = await transform(transformFunctionPath, output, context);
+      expect(transformedOutput).toBe('HELLO NAMED');
+    });
+
+    it('transforms output using a named function from a Python file', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      const pythonFilePath = 'file://transform.py:custom_transform';
+
+      const transformedOutput = await transform(pythonFilePath, output, context);
+      expect(transformedOutput).toBe('HELLO FROM PYTHON');
+      expect(runPython).toHaveBeenCalledWith(
+        expect.stringContaining('transform.py'),
+        'custom_transform',
+        [output, expect.any(Object)],
+      );
+    });
+
+    it('falls back to get_transform for Python files when no function name is provided', async () => {
+      const output = 'hello';
+      const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+      const pythonFilePath = 'file://transform.py';
+
+      const transformedOutput = await transform(pythonFilePath, output, context);
+      expect(transformedOutput).toBe('HELLO FROM PYTHON');
+      expect(runPython).toHaveBeenCalledWith(
+        expect.stringContaining('transform.py'),
+        'get_transform',
+        [output, expect.any(Object)],
+      );
+    });
+
+    it('does not throw error when validateReturn is false and function returns undefined', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const transformFunction = ''; // Empty function, returns undefined
+      const result = await transform(transformFunction, output, context, false);
+      expect(result).toBeUndefined();
+    });
+
+    it('throws error when validateReturn is true and function returns undefined', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const transformFunction = ''; // Empty function, returns undefined
+      await expect(transform(transformFunction, output, context, true)).rejects.toThrow(
+        'Transform function did not return a value',
+      );
+    });
+
+    it('does not throw error when validateReturn is false and function returns null', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const transformFunction = 'null'; // Will be wrapped with "return" automatically
+      const result = await transform(transformFunction, output, context, false);
+      expect(result).toBeNull();
+    });
+
+    it('throws error when validateReturn is true and function returns null', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const transformFunction = 'null'; // Will be wrapped with "return" automatically
+      await expect(transform(transformFunction, output, context, true)).rejects.toThrow(
+        'Transform function did not return a value',
+      );
+    });
+
+    it('handles file transform function errors gracefully', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const errorMessage = 'File not found';
+
+      mockedImportModule.mockRejectedValueOnce(new Error(errorMessage));
+
+      const transformFunctionPath = 'file://transform.js';
+      await expect(transform(transformFunctionPath, output, context)).rejects.toThrow(errorMessage);
+      expect(logger.error).toHaveBeenCalledWith(
+        'Error loading transform function from file',
+        expect.objectContaining({
+          message: errorMessage,
+          transform: FILE_TRANSFORM_LABEL,
+        }),
+      );
+    });
+
+    it('handles inline transform function errors gracefully', async () => {
+      const output = 'test';
+      const context = { vars: {}, prompt: {} };
+      const invalidFunction = 'invalid javascript code {';
+
+      await expect(transform(invalidFunction, output, context)).rejects.toThrow(
+        'Unexpected identifier',
+      );
+      expect(logger.error).toHaveBeenCalledWith(
+        'Error creating inline transform function',
+        expect.objectContaining({
+          transform: expect.stringContaining(INLINE_STRING_LABEL),
+        }),
+      );
+    });
+
+    describe('ESM compatibility', () => {
+      it('provides process.mainModule.require for backwards compatibility with CommonJS patterns', async () => {
+        const output = 'test';
+        const context = { vars: {}, prompt: {} };
+        // This is the exact pattern users were using before ESM migration
+        // const require = process.mainModule.require;
+        const transformFunction = `
+          const require = process.mainModule.require;
+          const path = require('node:path');
+          return path.basename('/foo/bar/baz.txt');
+        `;
+        const result = await transform(transformFunction, output, context);
+        expect(result).toBe('baz.txt');
+      });
+
+      it('allows direct use of process.mainModule.require without assignment', async () => {
+        const output = 'hello';
+        const context = { vars: {}, prompt: {} };
+        const transformFunction = `
+          const fs = process.mainModule.require('node:fs');
+          const os = process.mainModule.require('node:os');
+          return typeof fs.existsSync === 'function' && typeof os.homedir === 'function' ? output.toUpperCase() : output;
+        `;
+        const result = await transform(transformFunction, output, context);
+        expect(result).toBe('HELLO');
+      });
+
+      it('preserves other process properties like process.env', async () => {
+        const output = 'test';
+        const context = { vars: {}, prompt: {} };
+        const transformFunction = `
+          // process.env should still work
+          const env = process.env;
+          return typeof env === 'object' ? 'env-works' : 'env-broken';
+        `;
+        const result = await transform(transformFunction, output, context);
+        expect(result).toBe('env-works');
+      });
+    });
+
+    describe('file path handling', () => {
+      it('handles absolute paths in transform files', async () => {
+        const output = 'hello';
+        const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+
+        mockedImportModule.mockResolvedValueOnce((output: string) => output.toUpperCase());
+
+        const transformFunctionPath = 'file://transform.js';
+        const transformedOutput = await transform(transformFunctionPath, output, context);
+        expect(transformedOutput).toBe('HELLO');
+      });
+
+      it('handles file URLs in transform files', async () => {
+        const output = 'hello';
+        const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+
+        mockedImportModule.mockResolvedValueOnce((output: string) => output.toUpperCase());
+
+        const transformFunctionPath = 'file://transform.js';
+        const transformedOutput = await transform(transformFunctionPath, output, context);
+        expect(transformedOutput).toBe('HELLO');
+      });
+
+      it('handles Python files with absolute paths', async () => {
+        const output = 'hello';
+        const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+        const pythonFilePath = 'file://transform.py';
+
+        const transformedOutput = await transform(pythonFilePath, output, context);
+        expect(transformedOutput).toBe('HELLO FROM PYTHON');
+        expect(runPython).toHaveBeenCalledWith(
+          expect.stringContaining('transform.py'),
+          'get_transform',
+          [output, expect.any(Object)],
+        );
+      });
+
+      it('handles complex nested paths', async () => {
+        const output = 'hello';
+        const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+
+        mockedImportModule.mockResolvedValueOnce((output: string) => output.toUpperCase());
+
+        const transformFunctionPath = 'file://deeply/nested/path/with spaces/transform.js';
+        const transformedOutput = await transform(transformFunctionPath, output, context);
+        expect(transformedOutput).toBe('HELLO');
+      });
+
+      it('handles paths with special characters', async () => {
+        const output = 'hello';
+        const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+
+        mockedImportModule.mockResolvedValueOnce((output: string) => output.toUpperCase());
+
+        const transformFunctionPath = 'file://path/with-hyphens/and_underscores/transform.js';
+        const transformedOutput = await transform(transformFunctionPath, output, context);
+        expect(transformedOutput).toBe('HELLO');
+      });
+    });
+
+    describe('inline function transforms (Node.js package)', () => {
+      it('transforms output using a directly passed function', async () => {
+        const output = 'hello world';
+        const context = { vars: { key: 'value' }, prompt: { id: '123' } };
+        const fn: TransformFunction = (output) => String(output).toUpperCase();
+
+        const result = await transform(fn, output, context);
+        expect(result).toBe('HELLO WORLD');
+      });
+
+      it('transforms output using an async function', async () => {
+        const output = 'hello';
+        const context = { vars: {}, prompt: {} };
+        const fn: TransformFunction = async (output) => {
+          return output + ' async';
+        };
+
+        const result = await transform(fn, output, context);
+        expect(result).toBe('hello async');
+      });
+
+      it('passes context to the function', async () => {
+        const output = 'test';
+        const context = { vars: { name: 'Alice' }, prompt: { label: 'greeting' } };
+        const fn: TransformFunction = (output, ctx: any) => `${output} for ${ctx.vars.name}`;
+
+        const result = await transform(fn, output, context);
+        expect(result).toBe('test for Alice');
+      });
+
+      it('transforms vars using a directly passed function', async () => {
+        const vars = { key: 'value', other: 'data' };
+        const context = { vars: {}, prompt: {} };
+        const fn: TransformFunction = (vars: any) => ({ ...vars, key: vars.key.toUpperCase() });
+
+        const result = await transform(fn, vars, context, true, TransformInputType.VARS);
+        expect(result).toEqual({ key: 'VALUE', other: 'data' });
+      });
+
+      it('throws when inline function returns undefined and validateReturn is true', async () => {
+        const output = 'test';
+        const context = { vars: {}, prompt: {} };
+        const fn: TransformFunction = () => undefined as any;
+
+        await expect(transform(fn, output, context, true)).rejects.toThrow(
+          'Transform function did not return a value',
+        );
+      });
+
+      it('does not throw when inline function returns undefined and validateReturn is false', async () => {
+        const output = 'test';
+        const context = { vars: {}, prompt: {} };
+        const fn: TransformFunction = () => undefined as any;
+
+        const result = await transform(fn, output, context, false);
+        expect(result).toBeUndefined();
+      });
+
+      it('handles function that accesses metadata from context', async () => {
+        const output = 'raw output';
+        const context = {
+          vars: {},
+          prompt: {},
+          metadata: { toolCalls: [{ name: 'search' }, { name: 'calculate' }] },
+        };
+        const fn: TransformFunction = (_output, ctx: any) => {
+          const tools = ctx.metadata?.toolCalls ?? [];
+          return tools.map((t: any) => t.name).join(', ');
+        };
+
+        const result = await transform(fn, output, context);
+        expect(result).toBe('search, calculate');
+      });
+
+      it('propagates errors thrown by inline functions', async () => {
+        const fn: TransformFunction = () => {
+          throw new Error('user transform error');
+        };
+
+        await expect(transform(fn, 'test', { vars: {}, prompt: {} })).rejects.toThrow(
+          'user transform error',
+        );
+        expect(logger.error).toHaveBeenCalledWith(
+          'Error in transform function',
+          expect.objectContaining({
+            message: 'user transform error',
+            transform: expect.stringContaining(INLINE_FUNCTION_LABEL),
+          }),
+        );
+      });
+
+      it('wraps thrown errors with the transform label and preserves the original via cause', async () => {
+        const original = new Error('raw user error');
+        const fn: TransformFunction = () => {
+          throw original;
+        };
+
+        try {
+          await transform(fn, 'test', { vars: {}, prompt: {} });
+          throw new Error('expected transform to throw');
+        } catch (err) {
+          expect(err).toBeInstanceOf(Error);
+          const e = err as Error & { cause?: unknown };
+          // Wrapped message includes the label AND the original error text.
+          expect(e.message).toContain('Transform failed (');
+          expect(e.message).toContain(INLINE_FUNCTION_LABEL);
+          expect(e.message).toContain('raw user error');
+          // The original error is attached via the standard `cause` chain.
+          expect(e.cause).toBe(original);
+        }
+      });
+
+      it('resolves thenables returned by inline functions', async () => {
+        // Bare thenable built via Object.defineProperty to dodge the `noThenProperty`
+        // lint rule; verifies `transform()` awaits anything Promise-like.
+        const thenable = Object.defineProperty({}, 'then', {
+          value: (resolve: (value: unknown) => void) => resolve('thenable-resolved'),
+          enumerable: true,
+        }) as unknown as Promise<string>;
+        const fn: TransformFunction = () => thenable;
+
+        const result = await transform(fn, 'ignored', { vars: {}, prompt: {} });
+        expect(result).toBe('thenable-resolved');
+      });
+    });
+
+    describe('getTransformErrorMessage', () => {
+      it('returns the raw message from a transform-wrapped Error via .cause', () => {
+        const raw = new Error('underlying boom');
+        const wrapped = new Error('Transform failed ([inline function]: fn): underlying boom');
+        (wrapped as Error & { cause?: unknown }).cause = raw;
+        expect(getTransformErrorMessage(wrapped)).toBe('underlying boom');
+      });
+
+      it('falls back to the error message when there is no cause', () => {
+        expect(getTransformErrorMessage(new Error('plain'))).toBe('plain');
+      });
+
+      it('stringifies non-Error throw values', () => {
+        expect(getTransformErrorMessage('just a string')).toBe('just a string');
+        expect(getTransformErrorMessage(42)).toBe('42');
+      });
+
+      it('returns the outer message when the cause is not an Error', () => {
+        const wrapped = new Error('outer');
+        (wrapped as Error & { cause?: unknown }).cause = 'a string cause';
+        expect(getTransformErrorMessage(wrapped)).toBe('outer');
+      });
+    });
+
+    describe('getTransformLabel', () => {
+      it('shows inline string transforms verbatim', () => {
+        expect(getTransformLabel('output.toUpperCase()')).toBe(
+          `${INLINE_STRING_LABEL}: output.toUpperCase()`,
+        );
+      });
+
+      it('collapses whitespace in multi-line inline strings', () => {
+        expect(getTransformLabel('const x = output;\n  return x.trim();')).toBe(
+          `${INLINE_STRING_LABEL}: const x = output; return x.trim();`,
+        );
+      });
+
+      it('truncates long inline string transforms', () => {
+        const longExpr = 'output.' + 'a'.repeat(200);
+        const label = getTransformLabel(longExpr);
+        expect(label.startsWith(`${INLINE_STRING_LABEL}: `)).toBe(true);
+        expect(label.endsWith('…')).toBe(true);
+        expect(label.length).toBeLessThan(longExpr.length);
+      });
+
+      it('does not truncate an inline string exactly at the max length', () => {
+        const exprAtLimit = 'a'.repeat(80);
+        expect(getTransformLabel(exprAtLimit)).toBe(`${INLINE_STRING_LABEL}: ${exprAtLimit}`);
+      });
+
+      it('truncates an inline string one character over the max length', () => {
+        const exprOverLimit = 'a'.repeat(81);
+        const label = getTransformLabel(exprOverLimit);
+        // Label body is 79 chars + ellipsis → total 80 chars after the prefix.
+        expect(label).toBe(`${INLINE_STRING_LABEL}: ${'a'.repeat(79)}…`);
+      });
+
+      it('redacts file transform paths', () => {
+        expect(getTransformLabel('file://transform.js')).toBe(FILE_TRANSFORM_LABEL);
+      });
+
+      it('does not render function source in error labels', async () => {
+        const secretFn = function SECRET_SHOULD_NOT_APPEAR_IN_LABEL() {
+          return undefined;
+        };
+        await expect(transform(secretFn as any, 'test', { vars: {}, prompt: {} })).rejects.toThrow(
+          `Transform function did not return a value\n\n${INLINE_FUNCTION_LABEL}: SECRET_SHOULD_NOT_APPEAR_IN_LABEL`,
+        );
+      });
+
+      it('returns [inline function] for anonymous functions', () => {
+        expect(getTransformLabel(() => 'x')).toBe(INLINE_FUNCTION_LABEL);
+      });
+
+      it('includes function name for named functions', () => {
+        function myTransform() {
+          return 'x';
+        }
+        expect(getTransformLabel(myTransform)).toBe(`${INLINE_FUNCTION_LABEL}: myTransform`);
+      });
+
+      it('falls back to inline label for null or undefined inputs', () => {
+        expect(getTransformLabel(undefined)).toBe(INLINE_STRING_LABEL);
+        expect(getTransformLabel(null)).toBe(INLINE_STRING_LABEL);
+      });
+    });
+  });
+});

test/util/utils.test.ts

@@ -0,0 +1,139 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { mockGlobal, mockProcessEnv } from './utils';
+
+const envKeys = [
+  'PROMPTFOO_TEST_UTIL_ORIGINAL',
+  'PROMPTFOO_TEST_UTIL_OVERRIDE',
+  'PROMPTFOO_TEST_UTIL_CREATED',
+];
+
+const originalEnv = Object.fromEntries(envKeys.map((key) => [key, process.env[key]]));
+const testGlobalName = '__PROMPTFOO_TEST_UTIL_GLOBAL__';
+
+function restoreTestEnv(): void {
+  for (const key of envKeys) {
+    const value = originalEnv[key];
+    if (value === undefined) {
+      mockProcessEnv({ [key]: undefined });
+    } else {
+      mockProcessEnv({ [key]: value });
+    }
+  }
+}
+
+afterEach(() => {
+  restoreTestEnv();
+  Reflect.deleteProperty(globalThis, testGlobalName);
+  vi.doUnmock('node:fs');
+  vi.resetModules();
+  vi.clearAllMocks();
+});
+
+describe('mockProcessEnv', () => {
+  it('applies overrides and restores the previous environment snapshot', () => {
+    mockProcessEnv({ PROMPTFOO_TEST_UTIL_ORIGINAL: 'original' });
+
+    const restoreEnv = mockProcessEnv({
+      PROMPTFOO_TEST_UTIL_ORIGINAL: 'override',
+      PROMPTFOO_TEST_UTIL_OVERRIDE: 'set',
+    });
+
+    mockProcessEnv({ PROMPTFOO_TEST_UTIL_CREATED: 'created-by-test' });
+
+    expect(process.env.PROMPTFOO_TEST_UTIL_ORIGINAL).toBe('override');
+    expect(process.env.PROMPTFOO_TEST_UTIL_OVERRIDE).toBe('set');
+
+    restoreEnv();
+
+    expect(process.env.PROMPTFOO_TEST_UTIL_ORIGINAL).toBe('original');
+    expect(process.env.PROMPTFOO_TEST_UTIL_OVERRIDE).toBeUndefined();
+    expect(process.env.PROMPTFOO_TEST_UTIL_CREATED).toBeUndefined();
+  });
+
+  it('can start from an empty environment without replacing the process.env object', () => {
+    mockProcessEnv({ PROMPTFOO_TEST_UTIL_ORIGINAL: 'original' });
+    const envReference = process.env;
+
+    const restoreEnv = mockProcessEnv(
+      {
+        PROMPTFOO_TEST_UTIL_OVERRIDE: 'only-value',
+      },
+      { clear: true },
+    );
+
+    expect(process.env).toBe(envReference);
+    expect(process.env.PROMPTFOO_TEST_UTIL_ORIGINAL).toBeUndefined();
+    expect(process.env.PROMPTFOO_TEST_UTIL_OVERRIDE).toBe('only-value');
+
+    restoreEnv();
+
+    expect(process.env).toBe(envReference);
+    expect(process.env.PROMPTFOO_TEST_UTIL_ORIGINAL).toBe('original');
+    expect(process.env.PROMPTFOO_TEST_UTIL_OVERRIDE).toBeUndefined();
+  });
+});
+
+describe('mockGlobal', () => {
+  it('restores globals that did not previously exist', () => {
+    const restoreGlobal = mockGlobal(testGlobalName, { value: 'mocked' });
+
+    expect((globalThis as Record<string, unknown>)[testGlobalName]).toEqual({ value: 'mocked' });
+
+    restoreGlobal();
+
+    expect(testGlobalName in globalThis).toBe(false);
+  });
+
+  it('restores the previous global descriptor', () => {
+    Object.defineProperty(globalThis, testGlobalName, {
+      configurable: true,
+      get: () => 'original',
+    });
+
+    const restoreGlobal = mockGlobal(testGlobalName, 'mocked');
+
+    expect((globalThis as Record<string, unknown>)[testGlobalName]).toBe('mocked');
+
+    restoreGlobal();
+
+    expect((globalThis as Record<string, unknown>)[testGlobalName]).toBe('original');
+  });
+});
+
+describe('removeTempDir', () => {
+  it('retries transient recursive cleanup failures', async () => {
+    // Reset the module registry first so the dynamic import below re-evaluates
+    // ./utils against the mocked node:fs, even when this test runs in isolation
+    // (the top-level static import has already cached ./utils with the real fs).
+    vi.resetModules();
+    const rmSync = vi.fn();
+    vi.doMock('node:fs', async (importOriginal) => {
+      const actual = await importOriginal<typeof import('node:fs')>();
+      return { ...actual, rmSync };
+    });
+    const { removeTempDir } = await import('./utils');
+
+    removeTempDir('/tmp/promptfoo-test-dir');
+
+    expect(rmSync).toHaveBeenCalledWith('/tmp/promptfoo-test-dir', {
+      force: true,
+      maxRetries: 3,
+      recursive: true,
+      retryDelay: 100,
+    });
+  });
+
+  it('does not attempt cleanup without a directory', async () => {
+    vi.resetModules();
+    const rmSync = vi.fn();
+    vi.doMock('node:fs', async (importOriginal) => {
+      const actual = await importOriginal<typeof import('node:fs')>();
+      return { ...actual, rmSync };
+    });
+    const { removeTempDir } = await import('./utils');
+
+    removeTempDir(undefined);
+
+    expect(rmSync).not.toHaveBeenCalled();
+  });
+});

test/util/utils.ts

@@ -0,0 +1,185 @@
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import { vi } from 'vitest';
+import type { MockInstance } from 'vitest';
+
+import type { ApiProvider, ProviderResponse } from '../../src/types/index';
+
+/**
+ * Creates a deferred promise that can be resolved or rejected externally.
+ * Useful for controlling async flow in tests.
+ */
+export function createDeferred<T>(): {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+  reject: (reason?: unknown) => void;
+} {
+  let resolve!: (value: T) => void;
+  let reject!: (reason?: unknown) => void;
+  const promise = new Promise<T>((promiseResolve, promiseReject) => {
+    resolve = promiseResolve;
+    reject = promiseReject;
+  });
+  return { promise, resolve, reject };
+}
+
+export class TestGrader implements ApiProvider {
+  async callApi(): Promise<ProviderResponse> {
+    return {
+      output: JSON.stringify({ pass: true, reason: 'Test grading output' }),
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    };
+  }
+
+  id(): string {
+    return 'TestGradingProvider';
+  }
+}
+
+export function createMockResponse(
+  options: {
+    ok?: boolean;
+    body?: any;
+    statusText?: string;
+    status?: number;
+    headers?: Headers;
+    text?: () => Promise<string>;
+    json?: () => Promise<any>;
+  } = { ok: true },
+): Response {
+  const isOk = options.ok ?? (options.status ? options.status < 400 : true);
+  const mockResponse: Response = {
+    ok: isOk,
+    status: options.status || (isOk ? 200 : 400),
+    statusText: options.statusText || (isOk ? 'OK' : 'Bad Request'),
+    headers: options.headers || new Headers(),
+    redirected: false,
+    type: 'basic',
+    url: 'https://example.com',
+    json: options.json || (() => Promise.resolve(options.body || {})),
+    text: options.text || (() => Promise.resolve('')),
+    blob: () => Promise.resolve(new Blob([])),
+    arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)),
+    formData: () => Promise.resolve(new FormData()),
+    bodyUsed: false,
+    body: null,
+    clone() {
+      return createMockResponse(options);
+    },
+  } as Response;
+  return mockResponse;
+}
+
+export function stripAnsi(value: string): string {
+  return value.replace(/\u001b\[[0-9;]*m/g, '');
+}
+
+export type ConsoleMethod = 'debug' | 'error' | 'info' | 'log' | 'warn';
+
+export function mockConsole(
+  method: ConsoleMethod,
+  implementation: (...args: unknown[]) => void = () => {},
+): MockInstance {
+  return vi.spyOn(console, method).mockImplementation(implementation);
+}
+
+function replaceProcessEnv(nextEnv: Record<string, string | undefined>): void {
+  for (const key of Object.keys(process.env)) {
+    Reflect.deleteProperty(process.env, key);
+  }
+  Object.assign(process.env, nextEnv);
+}
+
+export function mockProcessEnv(
+  overrides: Record<string, string | undefined> = {},
+  options: { clear?: boolean } = {},
+): () => void {
+  const originalEnv = { ...process.env };
+
+  if (options.clear) {
+    replaceProcessEnv({});
+  }
+
+  for (const [key, value] of Object.entries(overrides)) {
+    if (value === undefined) {
+      Reflect.deleteProperty(process.env, key);
+    } else {
+      Object.assign(process.env, { [key]: value });
+    }
+  }
+
+  return () => {
+    replaceProcessEnv(originalEnv);
+  };
+}
+
+export const PROXY_ENV_KEYS = [
+  'HTTP_PROXY',
+  'http_proxy',
+  'HTTPS_PROXY',
+  'https_proxy',
+  'ALL_PROXY',
+  'all_proxy',
+  'NO_PROXY',
+  'no_proxy',
+  'NPM_CONFIG_PROXY',
+  'NPM_CONFIG_HTTP_PROXY',
+  'NPM_CONFIG_HTTPS_PROXY',
+  'npm_config_proxy',
+  'npm_config_http_proxy',
+  'npm_config_https_proxy',
+] as const;
+
+export function clearProxyEnv(): void {
+  for (const key of PROXY_ENV_KEYS) {
+    Reflect.deleteProperty(process.env, key);
+  }
+}
+
+export function mockGlobal<T>(name: string, value: T): () => void {
+  const descriptor = Object.getOwnPropertyDescriptor(globalThis, name);
+
+  Object.defineProperty(globalThis, name, {
+    configurable: true,
+    value,
+    writable: true,
+  });
+
+  return () => {
+    if (descriptor) {
+      Object.defineProperty(globalThis, name, descriptor);
+    } else {
+      Reflect.deleteProperty(globalThis, name);
+    }
+  };
+}
+
+/**
+ * Creates a unique temporary directory in the operating system temp location.
+ *
+ * @param prefix - Optional directory name prefix. Defaults to `promptfoo-test-`.
+ * @returns The absolute path to the newly created temporary directory.
+ */
+export function createTempDir(prefix = 'promptfoo-test-'): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+}
+
+/**
+ * Removes a temporary directory created during tests.
+ *
+ * If `tempDir` is undefined, this function is a no-op. Otherwise, it retries recursive removal
+ * briefly to tolerate transient file-handle contention on Windows.
+ *
+ * @param tempDir - Absolute or relative path to the temporary directory to remove.
+ */
+export function removeTempDir(tempDir: string | undefined): void {
+  if (!tempDir) {
+    return;
+  }
+  // On Windows, file handles can linger briefly after a stream is closed, so an
+  // immediate recursive delete may throw ENOTEMPTY/EBUSY/EPERM. Retry with a short
+  // backoff (a no-op on POSIX, where these errors don't occur).
+  fs.rmSync(tempDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
+}

test/util/validateTestProviderReferences.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, it } from 'vitest';
+import {
+  ProviderReferenceValidationError,
+  validateTestProviderReferences,
+} from '../../src/util/validateTestProviderReferences';
+import { createMockProvider } from '../factories/provider';
+
+import type { Scenario, TestCase } from '../../src/types/index';
+import type { ApiProvider } from '../../src/types/providers';
+
+describe('validateTestProviderReferences', () => {
+  const createProvider = (id: string, label?: string): ApiProvider =>
+    createMockProvider({ id, label });
+
+  const providers = [
+    createProvider('openai:gpt-4', 'smart-model'),
+    createProvider('openai:gpt-3.5-turbo', 'fast-model'),
+  ];
+
+  it('passes with no providers filter', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' } }];
+    expect(() => validateTestProviderReferences(tests, providers)).not.toThrow();
+  });
+
+  it('passes with valid provider label', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' }, providers: ['smart-model'] }];
+    expect(() => validateTestProviderReferences(tests, providers)).not.toThrow();
+  });
+
+  it('passes with valid provider id', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' }, providers: ['openai:gpt-4'] }];
+    expect(() => validateTestProviderReferences(tests, providers)).not.toThrow();
+  });
+
+  it('passes with wildcard match', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' }, providers: ['openai:*'] }];
+    expect(() => validateTestProviderReferences(tests, providers)).not.toThrow();
+  });
+
+  it('passes with legacy prefix match', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' }, providers: ['openai'] }];
+    expect(() => validateTestProviderReferences(tests, providers)).not.toThrow();
+  });
+
+  it('throws for invalid provider reference', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' }, providers: ['nonexistent'] }];
+    expect(() => validateTestProviderReferences(tests, providers)).toThrow(
+      ProviderReferenceValidationError,
+    );
+    expect(() => validateTestProviderReferences(tests, providers)).toThrow(
+      /does not exist.*Available providers/,
+    );
+  });
+
+  it('throws for invalid provider reference with description', () => {
+    const tests: TestCase[] = [
+      { description: 'My test case', vars: { foo: 'bar' }, providers: ['nonexistent'] },
+    ];
+    expect(() => validateTestProviderReferences(tests, providers)).toThrow(/My test case/);
+  });
+
+  it('validates defaultTest providers', () => {
+    const tests: TestCase[] = [];
+    const defaultTest = { providers: ['smart-model'] };
+    expect(() => validateTestProviderReferences(tests, providers, defaultTest)).not.toThrow();
+  });
+
+  it('throws for invalid defaultTest provider reference', () => {
+    const tests: TestCase[] = [];
+    const defaultTest = { providers: ['nonexistent'] };
+    expect(() => validateTestProviderReferences(tests, providers, defaultTest)).toThrow(
+      /defaultTest references provider/,
+    );
+  });
+
+  it('throws for non-array providers field', () => {
+    const tests: TestCase[] = [{ vars: { foo: 'bar' }, providers: 'not-an-array' as any }];
+    expect(() => validateTestProviderReferences(tests, providers)).toThrow(/must be an array/);
+  });
+
+  it('validates multiple tests', () => {
+    const tests: TestCase[] = [
+      { vars: { foo: 'bar' }, providers: ['smart-model'] },
+      { vars: { baz: 'qux' }, providers: ['fast-model'] },
+    ];
+    expect(() => validateTestProviderReferences(tests, providers)).not.toThrow();
+  });
+
+  it('throws for second invalid test', () => {
+    const tests: TestCase[] = [
+      { vars: { foo: 'bar' }, providers: ['smart-model'] },
+      { vars: { baz: 'qux' }, providers: ['nonexistent'] },
+    ];
+    expect(() => validateTestProviderReferences(tests, providers)).toThrow(/Test #2/);
+  });
+
+  describe('scenario validation', () => {
+    it('passes with valid scenario tests', () => {
+      const tests: TestCase[] = [];
+      const scenarios: Scenario[] = [
+        {
+          config: [{}],
+          tests: [{ vars: { foo: 'bar' }, providers: ['smart-model'] }],
+        },
+      ];
+      expect(() =>
+        validateTestProviderReferences(tests, providers, undefined, scenarios),
+      ).not.toThrow();
+    });
+
+    it('throws for invalid scenario test provider', () => {
+      const tests: TestCase[] = [];
+      const scenarios: Scenario[] = [
+        {
+          config: [{}],
+          tests: [{ vars: { foo: 'bar' }, providers: ['nonexistent'] }],
+        },
+      ];
+      expect(() => validateTestProviderReferences(tests, providers, undefined, scenarios)).toThrow(
+        /Scenario #1 test #1 references provider "nonexistent"/,
+      );
+    });
+
+    it('throws for invalid scenario config provider', () => {
+      const tests: TestCase[] = [];
+      const scenarios: Scenario[] = [
+        {
+          config: [{ providers: ['nonexistent'] }],
+          tests: [],
+        },
+      ];
+      expect(() => validateTestProviderReferences(tests, providers, undefined, scenarios)).toThrow(
+        /Scenario #1 config\[0\] references provider "nonexistent"/,
+      );
+    });
+
+    it('includes scenario description in error', () => {
+      const tests: TestCase[] = [];
+      const scenarios: Scenario[] = [
+        {
+          description: 'My scenario',
+          config: [{}],
+          tests: [{ vars: { foo: 'bar' }, providers: ['nonexistent'] }],
+        },
+      ];
+      expect(() => validateTestProviderReferences(tests, providers, undefined, scenarios)).toThrow(
+        /Scenario #1 \("My scenario"\)/,
+      );
+    });
+
+    it('validates multiple scenarios', () => {
+      const tests: TestCase[] = [];
+      const scenarios: Scenario[] = [
+        {
+          config: [{}],
+          tests: [{ vars: { foo: 'bar' }, providers: ['smart-model'] }],
+        },
+        {
+          config: [{}],
+          tests: [{ vars: { foo: 'bar' }, providers: ['nonexistent'] }],
+        },
+      ];
+      expect(() => validateTestProviderReferences(tests, providers, undefined, scenarios)).toThrow(
+        /Scenario #2/,
+      );
+    });
+  });
+});

test/util/verboseToggle.test.ts

@@ -0,0 +1,193 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('../../src/envars', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('../../src/envars')>()),
+  isCI: vi.fn(() => false),
+}));
+
+describe('verboseToggle', () => {
+  beforeEach(() => {
+    vi.resetModules();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should export initVerboseToggle function', async () => {
+    const { initVerboseToggle } = await import('../../src/util/verboseToggle');
+    expect(typeof initVerboseToggle).toBe('function');
+  });
+
+  it('should export disableVerboseToggle function', async () => {
+    const { disableVerboseToggle } = await import('../../src/util/verboseToggle');
+    expect(typeof disableVerboseToggle).toBe('function');
+  });
+
+  it('should export isVerboseToggleActive function', async () => {
+    const { isVerboseToggleActive } = await import('../../src/util/verboseToggle');
+    expect(typeof isVerboseToggleActive).toBe('function');
+  });
+
+  it('should return null when not in TTY mode', async () => {
+    // Save original values
+    const originalStdinIsTTY = process.stdin.isTTY;
+    const originalStdoutIsTTY = process.stdout.isTTY;
+
+    // Mock non-TTY mode
+    Object.defineProperty(process.stdin, 'isTTY', { value: false, configurable: true });
+    Object.defineProperty(process.stdout, 'isTTY', { value: false, configurable: true });
+
+    const { initVerboseToggle } = await import('../../src/util/verboseToggle');
+    const result = initVerboseToggle({ onInterrupt: vi.fn() });
+
+    expect(result).toBeNull();
+
+    // Restore original values
+    Object.defineProperty(process.stdin, 'isTTY', {
+      value: originalStdinIsTTY,
+      configurable: true,
+    });
+    Object.defineProperty(process.stdout, 'isTTY', {
+      value: originalStdoutIsTTY,
+      configurable: true,
+    });
+  });
+
+  it('should initially report toggle as inactive', async () => {
+    const { isVerboseToggleActive } = await import('../../src/util/verboseToggle');
+    // In non-TTY test environment, toggle won't be enabled
+    expect(isVerboseToggleActive()).toBe(false);
+  });
+
+  it('disableVerboseToggle should not throw when not initialized', async () => {
+    const { disableVerboseToggle } = await import('../../src/util/verboseToggle');
+    expect(() => disableVerboseToggle()).not.toThrow();
+  });
+
+  it('should fall back to process.exit(130) when onInterrupt throws so Ctrl+C is never broken', async () => {
+    const originalStdinIsTTY = process.stdin.isTTY;
+    const originalStdoutIsTTY = process.stdout.isTTY;
+    const originalSetRawMode = process.stdin.setRawMode;
+    const onInterrupt = vi.fn(() => {
+      throw new Error('caller-decided shutdown failed');
+    });
+    const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
+    const stderrWriteSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    Object.defineProperty(process.stdin, 'isTTY', { value: true, configurable: true });
+    Object.defineProperty(process.stdout, 'isTTY', { value: true, configurable: true });
+    Object.defineProperty(process.stdin, 'setRawMode', { value: vi.fn(), configurable: true });
+
+    try {
+      const { initVerboseToggle, isVerboseToggleActive } = await import(
+        '../../src/util/verboseToggle'
+      );
+
+      initVerboseToggle({ onInterrupt });
+      expect(() => process.stdin.emit('data', '\u0003')).not.toThrow();
+
+      expect(onInterrupt).toHaveBeenCalledTimes(1);
+      expect(exitSpy).toHaveBeenCalledWith(130);
+      expect(isVerboseToggleActive()).toBe(false);
+    } finally {
+      Object.defineProperty(process.stdin, 'isTTY', {
+        value: originalStdinIsTTY,
+        configurable: true,
+      });
+      Object.defineProperty(process.stdout, 'isTTY', {
+        value: originalStdoutIsTTY,
+        configurable: true,
+      });
+      Object.defineProperty(process.stdin, 'setRawMode', {
+        value: originalSetRawMode,
+        configurable: true,
+      });
+      stderrWriteSpy.mockRestore();
+      exitSpy.mockRestore();
+    }
+  });
+
+  it('should remove its exit listener during cleanup so init/teardown cycles do not leak', async () => {
+    const originalStdinIsTTY = process.stdin.isTTY;
+    const originalStdoutIsTTY = process.stdout.isTTY;
+    const originalSetRawMode = process.stdin.setRawMode;
+    const stderrWriteSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    Object.defineProperty(process.stdin, 'isTTY', { value: true, configurable: true });
+    Object.defineProperty(process.stdout, 'isTTY', { value: true, configurable: true });
+    Object.defineProperty(process.stdin, 'setRawMode', { value: vi.fn(), configurable: true });
+
+    try {
+      const { initVerboseToggle, disableVerboseToggle } = await import(
+        '../../src/util/verboseToggle'
+      );
+
+      const before = process.listenerCount('exit');
+      const cleanup = initVerboseToggle({ onInterrupt: vi.fn() });
+      expect(cleanup).not.toBeNull();
+      expect(process.listenerCount('exit')).toBe(before + 1);
+
+      disableVerboseToggle();
+      expect(process.listenerCount('exit')).toBe(before);
+    } finally {
+      Object.defineProperty(process.stdin, 'isTTY', {
+        value: originalStdinIsTTY,
+        configurable: true,
+      });
+      Object.defineProperty(process.stdout, 'isTTY', {
+        value: originalStdoutIsTTY,
+        configurable: true,
+      });
+      Object.defineProperty(process.stdin, 'setRawMode', {
+        value: originalSetRawMode,
+        configurable: true,
+      });
+      stderrWriteSpy.mockRestore();
+    }
+  });
+
+  it('should delegate Ctrl+C to the caller without exiting the process', async () => {
+    const originalStdinIsTTY = process.stdin.isTTY;
+    const originalStdoutIsTTY = process.stdout.isTTY;
+    const originalSetRawMode = process.stdin.setRawMode;
+    const onInterrupt = vi.fn();
+    const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
+    const stderrWriteSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    Object.defineProperty(process.stdin, 'isTTY', { value: true, configurable: true });
+    Object.defineProperty(process.stdout, 'isTTY', { value: true, configurable: true });
+    Object.defineProperty(process.stdin, 'setRawMode', {
+      value: vi.fn(),
+      configurable: true,
+    });
+
+    try {
+      const { initVerboseToggle, isVerboseToggleActive } = await import(
+        '../../src/util/verboseToggle'
+      );
+
+      initVerboseToggle({ onInterrupt });
+      process.stdin.emit('data', '\u0003');
+
+      expect(onInterrupt).toHaveBeenCalledTimes(1);
+      expect(exitSpy).not.toHaveBeenCalled();
+      expect(isVerboseToggleActive()).toBe(false);
+    } finally {
+      Object.defineProperty(process.stdin, 'isTTY', {
+        value: originalStdinIsTTY,
+        configurable: true,
+      });
+      Object.defineProperty(process.stdout, 'isTTY', {
+        value: originalStdoutIsTTY,
+        configurable: true,
+      });
+      Object.defineProperty(process.stdin, 'setRawMode', {
+        value: originalSetRawMode,
+        configurable: true,
+      });
+      stderrWriteSpy.mockRestore();
+      exitSpy.mockRestore();
+    }
+  });
+});

test/util/writeToFile.test.ts

@@ -0,0 +1,176 @@
+import { EventEmitter } from 'events';
+import type { WriteStream } from 'fs';
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { JsonlFileWriter } from '../../src/util/exportToFile/writeToFile';
+
+const mocks = vi.hoisted(() => ({
+  createWriteStream: vi.fn(),
+}));
+
+vi.mock('fs', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('fs')>()),
+  createWriteStream: mocks.createWriteStream,
+}));
+
+function createMockWriteStream({ destroyed = false }: { destroyed?: boolean } = {}) {
+  const stream = Object.assign(new EventEmitter(), {
+    closed: false,
+    destroyed,
+    end: vi.fn(),
+    // Mirror Node semantics: destroy() marks the stream destroyed and emits 'close'.
+    destroy: vi.fn(() => {
+      stream.destroyed = true;
+      stream.emit('close');
+      return stream;
+    }),
+    write: vi.fn((_data: string, callback?: (error?: Error | null) => void) => {
+      callback?.();
+      return true;
+    }),
+  });
+  mocks.createWriteStream.mockReturnValue(stream as unknown as WriteStream);
+  return stream;
+}
+
+describe('JsonlFileWriter', () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('opens the stream lazily and truncates by default, appending only when requested', async () => {
+    createMockWriteStream();
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    // Constructing the writer must not open (and thus truncate) the destination file.
+    expect(mocks.createWriteStream).not.toHaveBeenCalled();
+
+    await writer.write({ a: 1 });
+    expect(mocks.createWriteStream).toHaveBeenCalledWith('/tmp/results.jsonl', { flags: 'w' });
+
+    mocks.createWriteStream.mockClear();
+    createMockWriteStream();
+    const appender = new JsonlFileWriter('/tmp/results.jsonl', { append: true });
+    await appender.write({ a: 1 });
+    expect(mocks.createWriteStream).toHaveBeenCalledWith('/tmp/results.jsonl', { flags: 'a' });
+  });
+
+  it('leaves the destination untouched when nothing is written', async () => {
+    createMockWriteStream();
+    await expect(new JsonlFileWriter('/tmp/results.jsonl').close()).resolves.toBeUndefined();
+    expect(mocks.createWriteStream).not.toHaveBeenCalled();
+  });
+
+  it('waits for the underlying file descriptor to close', async () => {
+    const stream = createMockWriteStream();
+    stream.end.mockImplementation(() => stream);
+
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+    const closePromise = writer.close();
+    let settled = false;
+    closePromise.finally(() => {
+      settled = true;
+    });
+
+    await new Promise<void>((resolve) => setImmediate(resolve));
+    expect(settled).toBe(false);
+
+    stream.emit('close');
+    await expect(closePromise).resolves.toBeUndefined();
+    // Only the persistent recorder remains; close()'s temporary error listener was removed.
+    expect(stream.listenerCount('error')).toBe(1);
+  });
+
+  it('rejects errors emitted while the file descriptor is closing', async () => {
+    const stream = createMockWriteStream();
+    stream.end.mockImplementation(() => stream);
+
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+    const closePromise = writer.close();
+
+    stream.emit('error', new Error('late close error'));
+    stream.emit('close');
+
+    await expect(closePromise).rejects.toThrow(
+      'Failed to close JSONL output /tmp/results.jsonl: late close error',
+    );
+  });
+
+  it('resolves immediately when the stream already closed cleanly', async () => {
+    const stream = createMockWriteStream();
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+
+    stream.closed = true;
+    await expect(writer.close()).resolves.toBeUndefined();
+    expect(stream.end).not.toHaveBeenCalled();
+  });
+
+  it('rejects with a previously recorded error when the stream already closed', async () => {
+    const stream = createMockWriteStream();
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+
+    // A real stream that fails mid-run auto-destroys: 'error' and 'close' fire
+    // before close() is ever called. The recorded error must still reject close()
+    // rather than be swallowed (the output file is truncated/incomplete).
+    stream.emit('error', new Error('EBADF: bad file descriptor'));
+    stream.closed = true;
+
+    await expect(writer.close()).rejects.toThrow(
+      'Failed to close JSONL output /tmp/results.jsonl: EBADF: bad file descriptor',
+    );
+    expect(stream.end).not.toHaveBeenCalled();
+  });
+
+  it('waits for a destroyed stream to close without calling end()', async () => {
+    const stream = createMockWriteStream();
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+
+    stream.destroyed = true;
+    const closePromise = writer.close();
+    let settled = false;
+    closePromise.finally(() => {
+      settled = true;
+    });
+
+    await new Promise<void>((resolve) => setImmediate(resolve));
+    expect(settled).toBe(false);
+    expect(stream.end).not.toHaveBeenCalled();
+
+    stream.emit('close');
+    await expect(closePromise).resolves.toBeUndefined();
+  });
+
+  it('destroys the stream and rejects when end() throws synchronously', async () => {
+    const stream = createMockWriteStream();
+    stream.end.mockImplementation(() => {
+      throw new Error('end failed');
+    });
+
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+
+    await expect(writer.close()).rejects.toThrow(
+      'Failed to close JSONL output /tmp/results.jsonl: end failed',
+    );
+    expect(stream.destroy).toHaveBeenCalledTimes(1);
+  });
+
+  it('rejects final flush errors with the output path', async () => {
+    const stream = createMockWriteStream();
+    stream.end.mockImplementation(() => {
+      stream.emit('error', new Error('disk full'));
+      stream.emit('close');
+      return stream;
+    });
+
+    const writer = new JsonlFileWriter('/tmp/results.jsonl');
+    await writer.write({ a: 1 });
+    await expect(writer.close()).rejects.toThrow(
+      'Failed to close JSONL output /tmp/results.jsonl: disk full',
+    );
+  });
+});

test/util/writeToFileFs.test.ts

@@ -0,0 +1,66 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { JsonlFileWriter } from '../../src/util/exportToFile/writeToFile';
+
+// Unmocked counterpart to writeToFile.test.ts: pins the real-stream assumptions the
+// writer relies on — createWriteStream's default emitClose means 'close' always fires
+// (close() can never hang), and the file descriptor is released before close() resolves
+// so the file and its directory can be removed immediately (the Windows ENOTEMPTY race).
+describe('JsonlFileWriter (real filesystem)', () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-jsonl-'));
+  });
+
+  afterEach(() => {
+    fs.rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('releases the file descriptor before close() resolves', async () => {
+    const file = path.join(dir, 'out.jsonl');
+    const writer = new JsonlFileWriter(file);
+    await writer.write({ a: 1 });
+    await writer.write({ b: 2 });
+    await writer.close();
+
+    const lines = fs.readFileSync(file, 'utf-8').trim().split('\n');
+    expect(lines.map((line) => JSON.parse(line))).toEqual([{ a: 1 }, { b: 2 }]);
+
+    // Non-recursive removal fails (EBUSY/ENOTEMPTY on Windows) if the fd is still held.
+    fs.rmSync(file);
+    fs.rmdirSync(dir);
+    fs.mkdirSync(dir);
+  });
+
+  it('resolves a second close() call on an already-closed stream', async () => {
+    const writer = new JsonlFileWriter(path.join(dir, 'out.jsonl'));
+    await writer.write({ a: 1 });
+    await writer.close();
+    await expect(writer.close()).resolves.toBeUndefined();
+  });
+
+  it('leaves the destination untouched when nothing was written', async () => {
+    const file = path.join(dir, 'never.jsonl');
+    await expect(new JsonlFileWriter(file).close()).resolves.toBeUndefined();
+    expect(fs.existsSync(file)).toBe(false);
+  });
+
+  it('rejects close() when the stream failed mid-run', async () => {
+    const file = path.join(dir, 'broken.jsonl');
+    const writer = new JsonlFileWriter(file);
+    await writer.write({ a: 1 });
+    // Destroy with an error to simulate an fd failure between writes; the recorded
+    // pre-flush error must reject close() rather than be swallowed.
+    (writer as unknown as { writeStream: { destroy: (error: Error) => void } }).writeStream.destroy(
+      new Error('EBADF: bad file descriptor'),
+    );
+
+    await expect(writer.close()).rejects.toThrow(
+      `Failed to close JSONL output ${file}: EBADF: bad file descriptor`,
+    );
+  });
+});

test/util/xlsx.test.ts

@@ -0,0 +1,291 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { parseXlsxFile } from '../../src/util/xlsx';
+
+// Mock read-excel-file/node module
+const mockReadXlsxFile = vi.fn();
+
+vi.mock('read-excel-file/node', () => ({
+  __esModule: true,
+  default: (...args: unknown[]) => mockReadXlsxFile(...args),
+}));
+
+type SheetData = unknown[][];
+
+const createMockSheet = (sheet: string, data: SheetData = []) => ({ sheet, data });
+
+describe('parseXlsxFile', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('should parse xlsx file successfully', async () => {
+    // read-excel-file returns all sheets, each with an array of arrays where first row is headers
+    const mockRows = [
+      ['col1', 'col2'], // headers
+      ['value1', 'value2'], // data row 1
+      ['value3', 'value4'], // data row 2
+    ];
+
+    mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1', mockRows)]);
+
+    const result = await parseXlsxFile('test.xlsx');
+
+    expect(result).toEqual([
+      { col1: 'value1', col2: 'value2' },
+      { col1: 'value3', col2: 'value4' },
+    ]);
+    expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+  });
+
+  it('should throw error when read-excel-file module is not installed', async () => {
+    mockReadXlsxFile.mockRejectedValue(new Error("Cannot find module 'read-excel-file/node'"));
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow(
+      'read-excel-file is not installed. Please install it with: npm install read-excel-file',
+    );
+  });
+
+  it('should throw error when file parsing fails', async () => {
+    mockReadXlsxFile.mockRejectedValue(new Error('Failed to read file'));
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow(
+      'Failed to parse Excel file test.xlsx: Failed to read file',
+    );
+  });
+
+  it('should handle empty sheets', async () => {
+    mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1', [])]);
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow(
+      'Sheet "Sheet1" is empty or contains no valid data rows',
+    );
+  });
+
+  it('should throw error when Excel file has no sheets', async () => {
+    mockReadXlsxFile.mockResolvedValue([]);
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow('Excel file has no sheets');
+  });
+
+  it('should throw error when sheet has no valid column headers', async () => {
+    mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1', [['', '', '']])]);
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow(
+      'Sheet "Sheet1" has no valid column headers',
+    );
+  });
+
+  it('should throw error when sheet has headers but no data rows', async () => {
+    mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1', [['col1', 'col2']])]);
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow(
+      'Sheet "Sheet1" is empty or contains no valid data rows',
+    );
+  });
+
+  it('should throw error when sheet has headers but only empty data rows', async () => {
+    mockReadXlsxFile.mockResolvedValue([
+      createMockSheet('Sheet1', [
+        ['col1', 'col2'],
+        ['', ''],
+        [null, undefined],
+      ]),
+    ]);
+
+    await expect(parseXlsxFile('test.xlsx')).rejects.toThrow(
+      'Sheet "Sheet1" contains only empty data. Please ensure the sheet has both headers and data rows.',
+    );
+  });
+
+  it('should use first sheet by default', async () => {
+    const mockRows = [
+      ['col1', 'col2'],
+      ['value1', 'value2'],
+    ];
+
+    mockReadXlsxFile.mockResolvedValue([
+      createMockSheet('Sheet1', mockRows),
+      createMockSheet('Sheet2'),
+    ]);
+
+    const result = await parseXlsxFile('test.xlsx');
+
+    expect(result).toEqual([{ col1: 'value1', col2: 'value2' }]);
+    expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+  });
+
+  it('should handle malformed Excel files gracefully', async () => {
+    mockReadXlsxFile.mockRejectedValue(new Error('Invalid file format or corrupted file'));
+
+    await expect(parseXlsxFile('corrupted.xlsx')).rejects.toThrow(
+      'Failed to parse Excel file corrupted.xlsx: Invalid file format or corrupted file',
+    );
+  });
+
+  it('should throw specific error when file does not exist', async () => {
+    // read-excel-file surfaces ENOENT for missing files; parseXlsxFile rewraps it.
+    mockReadXlsxFile.mockRejectedValue(
+      Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' }),
+    );
+
+    await expect(parseXlsxFile('nonexistent.xlsx')).rejects.toThrow(
+      'File not found: nonexistent.xlsx',
+    );
+  });
+
+  describe('sheet selection syntax', () => {
+    it('should select sheet by name using # syntax', async () => {
+      const mockRows = [
+        ['col1', 'col2'],
+        ['sheet2data', 'value2'],
+      ];
+
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Sheet1'),
+        createMockSheet('DataSheet', mockRows),
+        createMockSheet('Sheet3'),
+      ]);
+
+      const result = await parseXlsxFile('test.xlsx#DataSheet');
+
+      expect(result).toEqual([{ col1: 'sheet2data', col2: 'value2' }]);
+      expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+    });
+
+    it('should select sheet by 1-based index using # syntax', async () => {
+      const mockRows = [
+        ['col1', 'col2'],
+        ['sheet2data', 'value2'],
+      ];
+
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Sheet1'),
+        createMockSheet('Sheet2', mockRows),
+        createMockSheet('Sheet3'),
+      ]);
+
+      const result = await parseXlsxFile('test.xlsx#2');
+
+      expect(result).toEqual([{ col1: 'sheet2data', col2: 'value2' }]);
+      expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+    });
+
+    it('should select sheet by index when the index has a leading plus sign', async () => {
+      const mockRows = [
+        ['col1', 'col2'],
+        ['sheet2data', 'value2'],
+      ];
+
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Sheet1'),
+        createMockSheet('Sheet2', mockRows),
+      ]);
+
+      const result = await parseXlsxFile('test.xlsx#+2');
+
+      expect(result).toEqual([{ col1: 'sheet2data', col2: 'value2' }]);
+      expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+    });
+
+    it('should throw error when sheet name does not exist', async () => {
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Sheet1'),
+        createMockSheet('Sheet2'),
+        createMockSheet('Sheet3'),
+      ]);
+
+      await expect(parseXlsxFile('test.xlsx#NonExistent')).rejects.toThrow(
+        'Sheet "NonExistent" not found. Available sheets: Sheet1, Sheet2, Sheet3',
+      );
+    });
+
+    it('should throw error when sheet index is out of bounds', async () => {
+      mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1'), createMockSheet('Sheet2')]);
+
+      await expect(parseXlsxFile('test.xlsx#5')).rejects.toThrow(
+        'Sheet index 5 is out of range. Available sheets: 2 (1-2)',
+      );
+    });
+
+    it('should throw error when sheet index is 0', async () => {
+      mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1'), createMockSheet('Sheet2')]);
+
+      await expect(parseXlsxFile('test.xlsx#0')).rejects.toThrow(
+        'Sheet index 0 is out of range. Available sheets: 2 (1-2)',
+      );
+    });
+
+    it('should throw error when sheet index is negative', async () => {
+      mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1'), createMockSheet('Sheet2')]);
+
+      await expect(parseXlsxFile('test.xlsx#-1')).rejects.toThrow(
+        'Sheet index -1 is out of range. Available sheets: 2 (1-2)',
+      );
+    });
+
+    it('should use first sheet when # is at end with no sheet specifier', async () => {
+      // Empty string after # is falsy, so it defaults to sheet 1
+      const mockRows = [
+        ['col1', 'col2'],
+        ['value1', 'value2'],
+      ];
+
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Sheet1', mockRows),
+        createMockSheet('Sheet2'),
+      ]);
+
+      const result = await parseXlsxFile('test.xlsx#');
+
+      expect(result).toEqual([{ col1: 'value1', col2: 'value2' }]);
+      expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+    });
+
+    it('should handle sheet names with spaces', async () => {
+      const mockRows = [
+        ['col1', 'col2'],
+        ['data', 'value'],
+      ];
+
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Sheet 1'),
+        createMockSheet('My Data Sheet', mockRows),
+        createMockSheet('Sheet 3'),
+      ]);
+
+      const result = await parseXlsxFile('test.xlsx#My Data Sheet');
+
+      expect(result).toEqual([{ col1: 'data', col2: 'value' }]);
+      expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+    });
+
+    it.each([
+      '2024-Q1',
+      '1H-results',
+      '2.9',
+    ])('should select sheet by name when the name starts with digits: %s', async (sheetName) => {
+      const mockRows = [
+        ['col1', 'col2'],
+        ['data', 'value'],
+      ];
+
+      mockReadXlsxFile.mockResolvedValue([
+        createMockSheet('Summary'),
+        createMockSheet(sheetName, mockRows),
+      ]);
+
+      const result = await parseXlsxFile(`test.xlsx#${sheetName}`);
+
+      expect(result).toEqual([{ col1: 'data', col2: 'value' }]);
+      expect(mockReadXlsxFile).toHaveBeenCalledWith('test.xlsx');
+    });
+
+    it('should throw not-found for a digit-prefixed name instead of out-of-range', async () => {
+      mockReadXlsxFile.mockResolvedValue([createMockSheet('Sheet1'), createMockSheet('Sheet2')]);
+
+      await expect(parseXlsxFile('test.xlsx#1H')).rejects.toThrow(
+        'Sheet "1H" not found. Available sheets: Sheet1, Sheet2',
+      );
+    });
+  });
+});

test/util/yamlLoad.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from 'vitest';
+import { loadYaml } from '../../src/util/yamlLoad';
+
+describe('loadYaml', () => {
+  it('parses plain YAML documents', () => {
+    expect(loadYaml('a: 1\nb: hello')).toEqual({ a: 1, b: 'hello' });
+  });
+
+  it('applies YAML merge keys like js-yaml v4', () => {
+    const content = [
+      'defaults: &defaults',
+      '  temperature: 0.5',
+      '  max_tokens: 100',
+      'provider:',
+      '  <<: *defaults',
+      '  max_tokens: 200',
+    ].join('\n');
+
+    expect(loadYaml(content)).toEqual({
+      defaults: { temperature: 0.5, max_tokens: 100 },
+      provider: { temperature: 0.5, max_tokens: 200 },
+    });
+  });
+
+  it('preserves js-yaml v4 legacy standard tags', () => {
+    expect(loadYaml('!!binary SGVsbG8=')).toEqual(new Uint8Array([72, 101, 108, 108, 111]));
+    expect(loadYaml('2024-01-02')).toEqual(new Date('2024-01-02T00:00:00.000Z'));
+    expect(loadYaml('!!omap [{a: 1}, {b: 2}]')).toEqual([{ a: 1 }, { b: 2 }]);
+    expect(loadYaml('!!pairs [{a: 1}, {b: 2}]')).toEqual([
+      ['a', 1],
+      ['b', 2],
+    ]);
+    expect(loadYaml('!!set {a: null, b: null}')).toEqual({ a: null, b: null });
+    expect(() => loadYaml('!!set {a: not-null}')).toThrow(/cannot resolve a set item/);
+  });
+
+  it('returns undefined for empty input like js-yaml v4', () => {
+    expect(loadYaml('')).toBeUndefined();
+  });
+
+  it('returns undefined for whitespace-only input', () => {
+    expect(loadYaml('   \n\t\n')).toBeUndefined();
+  });
+
+  it('returns undefined for comment-only input', () => {
+    expect(loadYaml('# just a comment\n# another comment\n')).toBeUndefined();
+  });
+
+  it('still throws on invalid YAML', () => {
+    expect(() => loadYaml('a: [unclosed')).toThrow();
+  });
+
+  it('still throws on multi-document streams like js-yaml v4 load', () => {
+    expect(() => loadYaml('a: 1\n---\nb: 2')).toThrow(/single document/);
+  });
+
+  it('passes through additional load options', () => {
+    expect(() => loadYaml('a: [unclosed', { filename: 'my-config.yaml' })).toThrow(
+      /my-config\.yaml/,
+    );
+  });
+});