chore: import upstream snapshot with attribution

test/evaluator/assertions.test.ts

@@ -0,0 +1,378 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import {
+  mockApiProvider,
+  mockGradingApiProviderFails,
+  mockGradingApiProviderPasses,
+  toPrompt,
+} from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator assertions', () => {
+  it('evaluate with expected value matching output', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].success).toBe(true);
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with expected value not matching output', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Different output',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(0);
+    expect(summary.stats.failures).toBe(1);
+    expect(summary.results[0].success).toBe(false);
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with fn: expected value', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'javascript',
+              value: 'output === "Test output";',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].success).toBe(true);
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with fn: expected value not matching output', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'javascript',
+              value: 'output === "Different output";',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(0);
+    expect(summary.stats.failures).toBe(1);
+    expect(summary.results[0].success).toBe(false);
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with grading expected value', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'llm-rubric',
+              value: 'output is a test output',
+            },
+          ],
+        },
+      ],
+      defaultTest: {
+        options: {
+          provider: mockGradingApiProviderPasses,
+        },
+      },
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].success).toBe(true);
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('reuses a configured LiteLLM provider ID for both G-Eval calls', async () => {
+    const configuredLiteLLM: ApiProvider = {
+      id: vi.fn().mockReturnValue('litellm:gemini-pro'),
+      config: { apiBaseUrl: 'http://localhost:4000', temperature: 0 },
+      callApi: vi.fn().mockImplementation(async (_prompt, context) => ({
+        output:
+          context?.prompt?.label === 'g-eval-steps'
+            ? JSON.stringify({ steps: ['Check factual accuracy'] })
+            : JSON.stringify({ score: 10, reason: 'The answer is accurate' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      })),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, configuredLiteLLM],
+      prompts: [toPrompt('What is the capital of France?')],
+      tests: [
+        {
+          providers: ['test-provider'],
+          assert: [
+            {
+              type: 'g-eval',
+              value: 'The answer identifies the capital correctly',
+              provider: 'litellm:gemini-pro',
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(configuredLiteLLM.callApi).toHaveBeenCalledTimes(2);
+    expect(
+      vi.mocked(configuredLiteLLM.callApi).mock.calls.map(([, context]) => context?.prompt?.label),
+    ).toEqual(['g-eval-steps', 'g-eval']);
+    expect(summary.stats.successes).toBe(1);
+  });
+
+  it('reuses configured graders in typed defaults and scenario assertion sets', async () => {
+    const configuredGrader: ApiProvider = {
+      id: vi.fn().mockReturnValue('litellm:judge'),
+      label: 'Configured grader',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'The output passes' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, configuredGrader],
+      prompts: [toPrompt('Test prompt')],
+      defaultTest: {
+        options: { provider: { text: 'Configured grader' } },
+      },
+      scenarios: [
+        {
+          config: [{}],
+          tests: [
+            {
+              providers: ['test-provider'],
+              assert: [
+                {
+                  type: 'assert-set',
+                  assert: [
+                    { type: 'llm-rubric', value: 'Use the default grader option' },
+                    {
+                      type: 'llm-rubric',
+                      value: 'Use the explicit configured grader ID option',
+                      provider: { text: { id: 'litellm:judge' } },
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(configuredGrader.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.stats.successes).toBe(1);
+  });
+
+  it('routes an assertion provider id to the id-matching provider when a sibling label collides', async () => {
+    const judgeById: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'Passes via id-matched judge.' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+    const judgeByLabel: ApiProvider = {
+      id: vi.fn().mockReturnValue('litellm:judge'),
+      label: 'judge',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ pass: false, score: 0, reason: 'Should never be called.' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, judgeById, judgeByLabel],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          providers: ['test-provider'],
+          assert: [{ type: 'llm-rubric', value: 'Use the id-matched judge', provider: 'judge' }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(judgeById.callApi).toHaveBeenCalledTimes(1);
+    expect(judgeByLabel.callApi).not.toHaveBeenCalled();
+    expect(summary.stats.successes).toBe(1);
+  });
+
+  it('preserves suite env for typed graders inherited from defaultTest assertions', async () => {
+    const testSuite: TestSuite = {
+      env: { LITELLM_API_BASE: 'echo' },
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{ providers: ['test-provider'] }],
+      defaultTest: {
+        assert: [
+          {
+            type: 'llm-rubric',
+            value: 'Grade the test output',
+            rubricPrompt: '{"pass":true,"score":1,"reason":"Resolved suite env"}',
+            provider: {
+              text: {
+                id: '{{ env.LITELLM_API_BASE }}',
+              },
+            },
+          },
+        ],
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.results[0].success).toBe(true);
+  });
+
+  it('reuses configured text and embedding graders for answer relevance', async () => {
+    const textJudge: ApiProvider = {
+      id: vi.fn().mockReturnValue('litellm:text-judge'),
+      callApi: vi.fn().mockResolvedValue({ output: 'Test prompt' }),
+    };
+    const embeddingJudge: ApiProvider = {
+      id: vi.fn().mockReturnValue('litellm:embedding:judge'),
+      callApi: vi.fn().mockResolvedValue({ output: '' }),
+      callEmbeddingApi: vi.fn().mockResolvedValue({ embedding: [1, 0] }),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, textJudge, embeddingJudge],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          providers: ['test-provider'],
+          assert: [
+            {
+              type: 'answer-relevance',
+              threshold: 0.8,
+              provider: {
+                text: 'litellm:text-judge',
+                embedding: 'litellm:embedding:judge',
+              },
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(textJudge.callApi).toHaveBeenCalledTimes(3);
+    expect(embeddingJudge.callEmbeddingApi).toHaveBeenCalledTimes(4);
+    expect(summary.stats.successes).toBe(1);
+  });
+
+  it('evaluate with grading expected value does not pass', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'llm-rubric',
+              value: 'output is a test output',
+            },
+          ],
+        },
+      ],
+      defaultTest: {
+        options: {
+          provider: mockGradingApiProviderFails,
+        },
+      },
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(0);
+    expect(summary.stats.failures).toBe(1);
+    expect(summary.results[0].success).toBe(false);
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+});

test/evaluator/basic.test.ts

@@ -0,0 +1,570 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+import fs from 'fs';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type TestSuite } from '../../src/types/index';
+import { processConfigFileReferences } from '../../src/util/fileReference';
+import { mockApiProvider, mockReasoningApiProvider, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator basic flows', () => {
+  it('evaluate with vars', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+      tests: [
+        {
+          vars: { var1: 'value1', var2: 'value2' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(mockApiProvider.callApi).toHaveBeenCalledWith(
+      'Test prompt value1 value2',
+      expect.objectContaining({
+        vars: expect.objectContaining({ var1: 'value1', var2: 'value2' }),
+        test: testSuite.tests![0],
+        prompt: expect.any(Object),
+      }),
+      undefined,
+    );
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+    expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with vars - no escaping', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+      tests: [
+        {
+          vars: { var1: '1 < 2', var2: 'he said "hello world"...' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt 1 < 2 he said "hello world"...');
+    expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with vars as object', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1.prop1 }} {{ var2 }}')],
+      tests: [
+        {
+          vars: { var1: { prop1: 'value1' }, var2: 'value2' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+    expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1.prop1 }} {{ var2 }}');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with vars from file', async () => {
+    const originalReadFileSync = fs.readFileSync;
+    vi.spyOn(fs, 'readFileSync').mockImplementation((path) => {
+      if (typeof path === 'string' && path.includes('test_file.txt')) {
+        return '<h1>Sample Report</h1><p>This is a test report with some data for the year 2023.</p>';
+      }
+      return originalReadFileSync(path);
+    });
+
+    const evalHelpers = await import('../../src/evaluatorHelpers');
+    const originalRenderPrompt = evalHelpers.renderPrompt;
+
+    const mockRenderPrompt = vi.spyOn(evalHelpers, 'renderPrompt');
+    mockRenderPrompt.mockImplementation(async (prompt, vars) => {
+      if (prompt.raw.includes('{{ var1 }}')) {
+        return 'Test prompt <h1>Sample Report</h1><p>This is a test report with some data for the year 2023.</p>';
+      }
+      return originalRenderPrompt(prompt, vars);
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1 }}')],
+      tests: [
+        {
+          vars: { var1: 'file://test/fixtures/test_file.txt' },
+        },
+      ],
+    };
+
+    try {
+      const processedTestSuite = await processConfigFileReferences(testSuite);
+      const evalRecord = await Eval.create({}, processedTestSuite.prompts, { id: randomUUID() });
+      await evaluate(processedTestSuite, evalRecord, {});
+      const summary = await evalRecord.toEvaluateSummary();
+
+      expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+      expect(mockApiProvider.callApi).toHaveBeenCalledWith(
+        'Test prompt <h1>Sample Report</h1><p>This is a test report with some data for the year 2023.</p>',
+        expect.anything(),
+        undefined,
+      );
+
+      expect(summary.stats.successes).toBe(1);
+      expect(summary.stats.failures).toBe(0);
+      expect(summary.results[0].prompt.raw).toBe(
+        'Test prompt <h1>Sample Report</h1><p>This is a test report with some data for the year 2023.</p>',
+      );
+      expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }}');
+      expect(summary.results[0].response?.output).toBe('Test output');
+    } finally {
+      mockRenderPrompt.mockRestore();
+      fs.readFileSync = originalReadFileSync;
+    }
+  });
+
+  it('evaluate with named prompt', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt {{ var1 }} {{ var2 }}', label: 'test display name' }],
+      tests: [
+        {
+          vars: { var1: 'value1', var2: 'value2' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+    expect(summary.results[0].prompt.label).toBe('test display name');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with multiple vars', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+      tests: [
+        {
+          vars: { var1: ['value1', 'value3'], var2: ['value2', 'value4'] },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+    expect(summary.stats.successes).toBe(4);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 40,
+      prompt: 20,
+      completion: 20,
+      cached: 0,
+      numRequests: 4,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+    expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with multiple providers', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+      tests: [
+        {
+          vars: { var1: 'value1', var2: 'value2' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.stats.successes).toBe(2);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 20,
+      prompt: 10,
+      completion: 10,
+      cached: 0,
+      numRequests: 2,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+    expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate without tests', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt');
+    expect(summary.results[0].prompt.label).toBe('Test prompt');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate without tests with multiple providers', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, mockApiProvider, mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
+    expect(summary.stats.successes).toBe(3);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 30,
+      prompt: 15,
+      completion: 15,
+      cached: 0,
+      numRequests: 3,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt');
+    expect(summary.results[0].prompt.label).toBe('Test prompt');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate for reasoning', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockReasoningApiProvider],
+      prompts: [toPrompt('Test prompt')],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockReasoningApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 21,
+      prompt: 9,
+      completion: 12,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 11,
+        acceptedPrediction: 12,
+        rejectedPrediction: 13,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt');
+    expect(summary.results[0].prompt.label).toBe('Test prompt');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+  it('allows test case repeat to override global repeat option', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          options: {
+            repeat: 3,
+          },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, {
+      id: randomUUID(),
+    });
+
+    await evaluate(testSuite, evalRecord, { repeat: 2 });
+
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
+    expect(
+      vi.mocked(mockApiProvider.callApi).mock.calls.map(([, context]) => context?.repeatIndex),
+    ).toEqual([0, 1, 2]);
+    expect(summary.results).toHaveLength(3);
+  });
+
+  it.each([
+    ['zero', 0],
+    ['negative', -1],
+    ['fractional', 1.5],
+    ['NaN', Number.NaN],
+    ['infinite', Number.POSITIVE_INFINITY],
+    ['unsafe', Number.MAX_SAFE_INTEGER + 1],
+  ])('falls back to global repeat for a %s per-test repeat', async (_label, repeat) => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          options: {
+            repeat,
+          },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, {
+      id: randomUUID(),
+    });
+
+    await evaluate(testSuite, evalRecord, { repeat: 2 });
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+  });
+});

test/evaluator/defaultTest.test.ts

@@ -0,0 +1,642 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { clearCache } from '../../src/cache';
+import cliState from '../../src/cliState';
+import { evaluate } from '../../src/evaluator';
+import { runExtensionHook } from '../../src/evaluatorHelpers';
+import { runDbMigrations } from '../../src/migrate';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import { createEmptyTokenUsage } from '../../src/util/tokenUsageUtils';
+import { mockApiProvider, resetMockProviders, toPrompt } from './helpers';
+
+afterEach(async () => {
+  resetMockProviders();
+  vi.mocked(runExtensionHook).mockReset();
+  vi.clearAllMocks();
+  cliState.resume = false;
+  cliState.basePath = '';
+  cliState.webUI = false;
+  await clearCache();
+});
+
+afterAll(() => {
+  vi.restoreAllMocks();
+  vi.resetModules();
+});
+
+describe('evaluator defaultTest merging', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    resetMockProviders();
+    // Reset runExtensionHook to default implementation (other tests may have overridden it)
+    vi.mocked(runExtensionHook).mockReset();
+    vi.mocked(runExtensionHook).mockImplementation(
+      async (_extensions, _hookName, context) => context,
+    );
+  });
+
+  it('should merge defaultTest.options.provider with test case options', async () => {
+    const mockProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('mock-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      prompts: [toPrompt('Test prompt {{text}}')],
+      providers: [mockProvider],
+      tests: [
+        {
+          vars: { text: 'Hello world' },
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+            },
+          ],
+        },
+      ],
+      defaultTest: {
+        options: {
+          provider: {
+            embedding: {
+              id: 'bedrock:embeddings:amazon.titan-embed-text-v2:0',
+              config: {
+                region: 'us-east-1',
+              },
+            },
+          },
+        },
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // The evaluator should have processed the tests and merged defaultTest options
+    expect(summary.results).toBeDefined();
+    expect(summary.results.length).toBeGreaterThan(0);
+
+    // Check that the test case has the merged options from defaultTest
+    const processedTest = summary.results[0].testCase;
+    expect(processedTest?.options?.provider).toEqual({
+      embedding: {
+        id: 'bedrock:embeddings:amazon.titan-embed-text-v2:0',
+        config: {
+          region: 'us-east-1',
+        },
+      },
+    });
+  });
+
+  it('should allow test case options to override defaultTest options', async () => {
+    const mockProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('mock-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      prompts: [toPrompt('Test prompt {{text}}')],
+      providers: [mockProvider],
+      tests: [
+        {
+          vars: { text: 'Hello world' },
+          options: {
+            provider: 'openai:gpt-4',
+          },
+          assert: [
+            {
+              type: 'llm-rubric',
+              value: 'Output is correct',
+            },
+          ],
+        },
+      ],
+      defaultTest: {
+        options: {
+          provider: 'openai:gpt-3.5-turbo',
+          transform: 'output.toUpperCase()',
+        },
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // Check that the test case options override defaultTest options
+    const processedTest = summary.results[0].testCase;
+    expect(processedTest?.options?.provider).toBe('openai:gpt-4');
+    // But other defaultTest options should still be merged
+    expect(processedTest?.options?.transform).toBe('output.toUpperCase()');
+  });
+});
+
+describe('Evaluator with external defaultTest', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // Reset runExtensionHook to default implementation (other tests may have overridden it)
+    vi.mocked(runExtensionHook).mockReset();
+    vi.mocked(runExtensionHook).mockImplementation(
+      async (_extensions, _hookName, context) => context,
+    );
+  });
+
+  it('should handle string defaultTest gracefully', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt {{var}}', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      defaultTest: 'file://path/to/defaultTest.yaml' as any, // String should have been resolved before reaching evaluator
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // Should handle gracefully even if string wasn't resolved
+    expect(summary.results).toHaveLength(1);
+    expect(summary.results[0].vars).toEqual({ var: 'value' });
+  });
+
+  it('should apply object defaultTest properties correctly', async () => {
+    const defaultTest = {
+      assert: [{ type: 'equals' as const, value: 'expected' }],
+      vars: { defaultVar: 'defaultValue' },
+      options: { provider: 'test-provider' },
+      metadata: { suite: 'test-suite' },
+      threshold: 0.8,
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [
+        { vars: { testVar: 'testValue' } },
+        {
+          vars: { testVar: 'override' },
+          assert: [{ type: 'contains' as const, value: 'exp' }],
+          threshold: 0.9,
+        },
+      ],
+      defaultTest,
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // First test should inherit all defaultTest properties
+    const firstResult = summary.results[0] as any;
+    expect(firstResult.testCase.assert).toEqual(defaultTest.assert);
+    expect(firstResult.testCase.vars).toEqual({
+      defaultVar: 'defaultValue',
+      testVar: 'testValue',
+    });
+    expect(firstResult.testCase.threshold).toBe(0.8);
+    expect(firstResult.testCase.metadata).toEqual({ suite: 'test-suite' });
+
+    // Second test should merge/override appropriately
+    const secondResult = summary.results[1] as any;
+    expect(secondResult.testCase.assert).toEqual([
+      ...defaultTest.assert,
+      { type: 'contains' as const, value: 'exp' },
+    ]);
+    expect(secondResult.testCase.threshold).toBe(0.9); // Override
+  });
+
+  it('should allow a test case to opt out of defaultTest assertions', async () => {
+    const defaultTest = {
+      assert: [{ type: 'equals' as const, value: 'expected' }],
+      vars: { defaultVar: 'defaultValue' },
+      options: { provider: 'default-provider' },
+      metadata: { suite: 'test-suite' },
+      threshold: 0.8,
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [
+        {
+          vars: { testVar: 'testValue' },
+          options: { disableDefaultAsserts: true },
+          assert: [{ type: 'contains' as const, value: 'exp' }],
+        },
+      ],
+      defaultTest,
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    const result = summary.results[0] as any;
+    expect(result.testCase.assert).toEqual([{ type: 'contains' as const, value: 'exp' }]);
+    expect(result.testCase.vars).toEqual({
+      defaultVar: 'defaultValue',
+      testVar: 'testValue',
+    });
+    expect(result.testCase.threshold).toBe(0.8);
+    expect(result.testCase.metadata).toEqual({ suite: 'test-suite' });
+    expect(result.testCase.options).toMatchObject({
+      provider: 'default-provider',
+      disableDefaultAsserts: true,
+    });
+  });
+
+  it('should handle invariant check for defaultTest.assert array', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      defaultTest: {
+        assert: 'not-an-array' as any, // Invalid type
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+    // Should throw or handle gracefully
+    await expect(evaluate(testSuite, evalRecord, {})).rejects.toThrow(
+      'defaultTest.assert is not an array in test case #1',
+    );
+  });
+
+  it('should correctly merge defaultTest with test case when defaultTest is object', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test {{var}}', label: 'test' }],
+      tests: [
+        {
+          vars: { var: 'test1' },
+          options: { transformVars: 'vars.transformed = true; return vars;' },
+        },
+      ],
+      defaultTest: {
+        vars: { defaultVar: 'default' },
+        options: {
+          provider: 'default-provider',
+          transformVars: 'vars.defaultTransform = true; return vars;',
+        },
+        assert: [{ type: 'not-equals' as const, value: '' }],
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // Test case transformVars should override defaultTest transformVars
+    const result = summary.results[0] as any;
+    expect(result.testCase.options?.transformVars).toBe('vars.transformed = true; return vars;');
+    // But other options should be merged
+    expect(result.testCase.options?.provider).toBe('default-provider');
+  });
+
+  it('should preserve metrics from existing prompts when resuming evaluation', async () => {
+    // Store original resume state and ensure it's false
+    const originalResume = cliState.resume;
+    cliState.resume = false;
+
+    try {
+      // Create a test suite with 2 prompts and 1 test
+      const testSuite: TestSuite = {
+        providers: [mockApiProvider],
+        prompts: [
+          { raw: 'Test prompt 1', label: 'test1' },
+          { raw: 'Test prompt 2', label: 'test2' },
+        ],
+        tests: [{ vars: { var: 'value1' } }],
+      };
+
+      // Create initial eval record
+      const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+      // Simulate that the eval was already partially completed with some metrics
+      const initialMetrics1 = {
+        score: 10,
+        testPassCount: 1,
+        testFailCount: 0,
+        testErrorCount: 0,
+        assertPassCount: 1,
+        assertFailCount: 0,
+        totalLatencyMs: 100,
+        tokenUsage: createEmptyTokenUsage(),
+        namedScores: {},
+        namedScoresCount: {},
+        cost: 0.001,
+      };
+
+      const initialMetrics2 = {
+        score: 5,
+        testPassCount: 0,
+        testFailCount: 1,
+        testErrorCount: 0,
+        assertPassCount: 0,
+        assertFailCount: 1,
+        totalLatencyMs: 150,
+        tokenUsage: createEmptyTokenUsage(),
+        namedScores: {},
+        namedScoresCount: {},
+        cost: 0.002,
+      };
+
+      evalRecord.prompts = [
+        {
+          raw: 'Test prompt 1',
+          label: 'test1',
+          id: 'prompt-test1',
+          provider: 'test-provider',
+          metrics: { ...initialMetrics1 },
+        },
+        {
+          raw: 'Test prompt 2',
+          label: 'test2',
+          id: 'prompt-test2',
+          provider: 'test-provider',
+          metrics: { ...initialMetrics2 },
+        },
+      ];
+      evalRecord.persisted = true;
+
+      // Enable resume mode
+      cliState.resume = true;
+
+      // Run evaluation with resume - this will run the test on both prompts
+      await evaluate(testSuite, evalRecord, {});
+
+      // Verify the prompts still exist and have the right IDs
+      expect(evalRecord.prompts).toHaveLength(2);
+      expect(evalRecord.prompts[0].id).toBe('prompt-test1');
+      expect(evalRecord.prompts[1].id).toBe('prompt-test2');
+
+      // Check that the prompts have preserved metrics
+      // When resuming, the metrics should be accumulated with the initial values
+      // The key test is that metrics are not reset to 0
+
+      // For prompt 1 which had testPassCount=1 initially
+      expect(evalRecord.prompts[0].metrics?.testPassCount).toBeGreaterThanOrEqual(1);
+
+      // For prompt 2, at least verify metrics exist and aren't completely reset
+      expect(evalRecord.prompts[1].metrics).toBeDefined();
+
+      // The combined pass/fail count should be greater than 0, showing metrics weren't reset
+      const prompt2TotalTests =
+        (evalRecord.prompts[1].metrics?.testPassCount || 0) +
+        (evalRecord.prompts[1].metrics?.testFailCount || 0);
+      expect(prompt2TotalTests).toBeGreaterThan(0);
+    } finally {
+      // Always restore original state
+      cliState.resume = originalResume;
+    }
+  });
+
+  it('should backfill legacy named score weights when resuming evaluation', async () => {
+    const originalResume = cliState.resume;
+    cliState.resume = false;
+
+    try {
+      const testSuite: TestSuite = {
+        providers: [mockApiProvider],
+        prompts: [{ raw: 'Test prompt 1', label: 'test1' }],
+        tests: [
+          {
+            assert: [
+              {
+                type: 'equals',
+                value: 'Test output',
+                metric: 'accuracy',
+                weight: 3,
+              },
+              {
+                type: 'contains',
+                value: 'Missing output',
+                metric: 'accuracy',
+                weight: 1,
+              },
+            ],
+          },
+        ],
+      };
+
+      const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+      evalRecord.prompts = [
+        {
+          raw: 'Test prompt 1',
+          label: 'test1',
+          id: 'prompt-test1',
+          provider: 'test-provider',
+          metrics: {
+            score: 1,
+            testPassCount: 1,
+            testFailCount: 0,
+            testErrorCount: 0,
+            assertPassCount: 1,
+            assertFailCount: 0,
+            totalLatencyMs: 100,
+            tokenUsage: createEmptyTokenUsage(),
+            namedScores: { accuracy: 1 },
+            namedScoresCount: { accuracy: 1 },
+            cost: 0.001,
+          },
+        },
+      ];
+      evalRecord.persisted = true;
+      cliState.resume = true;
+
+      await evaluate(testSuite, evalRecord, {});
+
+      expect(evalRecord.prompts[0].metrics?.namedScores.accuracy).toBeCloseTo(4, 10);
+      expect(evalRecord.prompts[0].metrics?.namedScoresCount.accuracy).toBe(3);
+      expect(evalRecord.prompts[0].metrics?.namedScoreWeights?.accuracy).toBe(5);
+    } finally {
+      cliState.resume = originalResume;
+    }
+  });
+});
+
+describe('defaultTest normalization for extensions', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // Reset runExtensionHook to default implementation (other tests may have overridden it)
+    vi.mocked(runExtensionHook).mockReset();
+    vi.mocked(runExtensionHook).mockImplementation(
+      async (_extensions, _hookName, context) => context,
+    );
+  });
+
+  it('should initialize defaultTest when undefined and extensions are present', async () => {
+    const mockExtension = 'file://test-extension.js';
+    let capturedSuite: TestSuite | undefined;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'beforeAll') {
+        capturedSuite = (context as { suite: TestSuite }).suite;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      extensions: [mockExtension],
+      // No defaultTest defined
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedSuite).toBeDefined();
+    expect(capturedSuite!.defaultTest).toBeDefined();
+    expect(capturedSuite!.defaultTest).toEqual({ assert: [] });
+  });
+
+  it('should initialize defaultTest.assert when defaultTest exists but assert is undefined', async () => {
+    const mockExtension = 'file://test-extension.js';
+    let capturedSuite: TestSuite | undefined;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'beforeAll') {
+        capturedSuite = (context as { suite: TestSuite }).suite;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      extensions: [mockExtension],
+      defaultTest: {
+        vars: { defaultVar: 'defaultValue' },
+        // No assert defined
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedSuite).toBeDefined();
+    expect(capturedSuite!.defaultTest).toBeDefined();
+    const defaultTest = capturedSuite!.defaultTest as Record<string, unknown>;
+    expect(defaultTest.vars).toEqual({ defaultVar: 'defaultValue' });
+    expect(defaultTest.assert).toEqual([]);
+  });
+
+  it('should preserve existing defaultTest.assert when extensions are present', async () => {
+    const mockExtension = 'file://test-extension.js';
+    let capturedSuite: TestSuite | undefined;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'beforeAll') {
+        capturedSuite = (context as { suite: TestSuite }).suite;
+      }
+      return context;
+    });
+
+    const existingAssertions = [
+      { type: 'contains' as const, value: 'expected' },
+      { type: 'not-contains' as const, value: 'unexpected' },
+    ];
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      extensions: [mockExtension],
+      defaultTest: {
+        assert: existingAssertions,
+      },
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedSuite).toBeDefined();
+    const defaultTest = capturedSuite!.defaultTest as Record<string, unknown>;
+    expect(defaultTest.assert).toBe(existingAssertions); // Same reference
+    expect(defaultTest.assert).toHaveLength(2);
+  });
+
+  it('should not modify defaultTest when no extensions are present', async () => {
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockClear();
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      // No extensions
+      // No defaultTest
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    // runExtensionHook should still be called (with empty/undefined extensions)
+    // but the beforeAll hook call should receive the original suite without normalization
+    const beforeAllCall = mockedRunExtensionHook.mock.calls.find((call) => call[1] === 'beforeAll');
+    expect(beforeAllCall).toBeDefined();
+    const suite = (beforeAllCall?.[2] as { suite: TestSuite } | undefined)?.suite;
+    expect(suite?.defaultTest).toBeUndefined();
+  });
+
+  it('should allow extensions to push to defaultTest.assert safely', async () => {
+    const mockExtension = 'file://test-extension.js';
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'beforeAll') {
+        // Simulate what an extension would do - push to assert array
+        // This should work because defaultTest.assert is guaranteed to be an array
+        const suite = (context as { suite: TestSuite }).suite;
+        const defaultTest = suite.defaultTest as Exclude<typeof suite.defaultTest, string>;
+        defaultTest!.assert!.push({ type: 'is-json' as const });
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [{ vars: { var: 'value' } }],
+      extensions: [mockExtension],
+      // No defaultTest - will be initialized by evaluator
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    // The assertion added by the extension should be present in the results
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary.results[0].testCase.assert).toContainEqual({ type: 'is-json' });
+  });
+});

test/evaluator/generatedPrompts.test.ts

@@ -0,0 +1,143 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { evaluate, PromptSuggestionsRejectedError } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { generatePrompts } from '../../src/suggestions';
+import { promptYesNo } from '../../src/util/readline';
+import { createEmptyTokenUsage } from '../../src/util/tokenUsageUtils';
+import { createMockProvider } from '../factories/provider';
+
+import type { TestSuite } from '../../src/types';
+
+vi.mock('../../src/suggestions', () => ({
+  generatePrompts: vi.fn(),
+}));
+
+vi.mock('../../src/util/readline', () => ({
+  promptYesNo: vi.fn(),
+}));
+
+describe('generated prompt selection', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(generatePrompts).mockResolvedValue({
+      prompts: ['Generated prompt'],
+      tokensUsed: createEmptyTokenUsage(),
+    });
+    vi.mocked(promptYesNo).mockResolvedValue(false);
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  function createTestSuite(): TestSuite {
+    return {
+      providers: [createMockProvider()],
+      prompts: [{ raw: 'Original prompt', label: 'Original prompt' }],
+      tests: [{}],
+    };
+  }
+
+  it('throws without mutating process.exitCode for reusable callers', async () => {
+    const previousExitCode = process.exitCode;
+    process.exitCode = undefined;
+
+    try {
+      await expect(
+        evaluate(createTestSuite(), new Eval({}), {
+          eventSource: 'library',
+          generateSuggestions: true,
+        }),
+      ).rejects.toEqual(expect.any(PromptSuggestionsRejectedError));
+
+      expect(process.exitCode).toBeUndefined();
+    } finally {
+      process.exitCode = previousExitCode;
+    }
+  });
+
+  it('preserves the CLI exit code when all suggested prompts are rejected', async () => {
+    const previousExitCode = process.exitCode;
+    process.exitCode = undefined;
+
+    try {
+      await evaluate(createTestSuite(), new Eval({}), {
+        eventSource: 'cli',
+        generateSuggestions: true,
+      });
+
+      expect(process.exitCode).toBe(1);
+    } finally {
+      process.exitCode = previousExitCode;
+    }
+  });
+
+  it('does not throw or mutate process.exitCode when the user accepts a suggestion', async () => {
+    const previousExitCode = process.exitCode;
+    process.exitCode = undefined;
+    vi.mocked(promptYesNo).mockResolvedValue(true);
+
+    try {
+      const testSuite = createTestSuite();
+      // Library mode would throw PromptSuggestionsRejectedError if a regression
+      // flipped the boolean — covers the accepted branch that the rejection
+      // tests above intentionally don't exercise.
+      await expect(
+        evaluate(testSuite, new Eval({}), {
+          eventSource: 'library',
+          generateSuggestions: true,
+        }),
+      ).resolves.toBeDefined();
+      expect(process.exitCode).toBeUndefined();
+    } finally {
+      process.exitCode = previousExitCode;
+    }
+  });
+
+  it('passes the requested suggestion count to the generator', async () => {
+    vi.mocked(promptYesNo).mockResolvedValueOnce(true);
+
+    await evaluate(createTestSuite(), new Eval({}), {
+      eventSource: 'library',
+      generateSuggestions: true,
+      suggestionsCount: 3,
+    });
+
+    expect(generatePrompts).toHaveBeenCalledWith('Original prompt', 3);
+  });
+
+  it('defaults suggestionsCount to 1 when omitted', async () => {
+    vi.mocked(promptYesNo).mockResolvedValueOnce(true);
+
+    await evaluate(createTestSuite(), new Eval({}), {
+      eventSource: 'library',
+      generateSuggestions: true,
+    });
+
+    expect(generatePrompts).toHaveBeenCalledWith('Original prompt', 1);
+  });
+
+  it('clamps over-cap suggestionsCount to MAX_SUGGESTIONS_COUNT', async () => {
+    vi.mocked(promptYesNo).mockResolvedValueOnce(true);
+
+    await evaluate(createTestSuite(), new Eval({}), {
+      eventSource: 'library',
+      generateSuggestions: true,
+      suggestionsCount: 1_000,
+    });
+
+    expect(generatePrompts).toHaveBeenCalledWith('Original prompt', 50);
+  });
+
+  it('coerces invalid suggestionsCount values to 1', async () => {
+    vi.mocked(promptYesNo).mockResolvedValueOnce(true);
+
+    await evaluate(createTestSuite(), new Eval({}), {
+      eventSource: 'library',
+      generateSuggestions: true,
+      suggestionsCount: 0,
+    });
+
+    expect(generatePrompts).toHaveBeenCalledWith('Original prompt', 1);
+  });
+});

test/evaluator/gradingConcurrency.test.ts

@@ -0,0 +1,952 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { clearCache, getCache } from '../../src/cache';
+import { evaluate, runEval } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import {
+  type ApiProvider,
+  type RateLimitRegistryRef,
+  ResultFailureReason,
+  type TestSuite,
+} from '../../src/types/index';
+import { createEmptyTokenUsage } from '../../src/util/tokenUsageUtils';
+import { toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator grading concurrency', () => {
+  it('schedules model-graded assertion provider calls through the rate limit registry', async () => {
+    const abortController = new AbortController();
+    const execute = vi.fn(async (_provider: ApiProvider, callFn: () => Promise<unknown>) =>
+      callFn(),
+    );
+    const rateLimitRegistry = {
+      execute,
+      dispose: vi.fn(),
+    } as RateLimitRegistryRef;
+    const targetProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test response',
+        tokenUsage: createEmptyTokenUsage(),
+      }),
+    };
+    const gradingProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('grading-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ pass: true, reason: 'Scheduled grading passed' }),
+        tokenUsage: createEmptyTokenUsage(),
+      }),
+    };
+
+    const results = await runEval({
+      delay: 0,
+      testIdx: 0,
+      promptIdx: 0,
+      repeatIndex: 0,
+      isRedteam: false,
+      provider: targetProvider,
+      prompt: { raw: 'Test prompt', label: 'test-label' },
+      test: {
+        assert: [
+          {
+            type: 'llm-rubric',
+            value: 'Output should be valid',
+            provider: gradingProvider,
+          },
+        ],
+      },
+      conversations: {},
+      registers: {},
+      abortSignal: abortController.signal,
+      rateLimitRegistry,
+    });
+
+    expect(results[0].success).toBe(true);
+    expect(execute).toHaveBeenCalledTimes(2);
+    expect(execute.mock.calls.map(([provider]) => provider.id())).toEqual([
+      'target-provider',
+      'grading-provider',
+    ]);
+    expect(gradingProvider.callApi).toHaveBeenCalledWith(
+      expect.stringContaining('Output should be valid'),
+      expect.objectContaining({
+        prompt: { raw: expect.any(String), label: 'llm-rubric' },
+        vars: expect.objectContaining({
+          output: 'Test response',
+          rubric: 'Output should be valid',
+        }),
+      }),
+      { abortSignal: abortController.signal },
+    );
+  });
+
+  it('groups model-graded assertion calls by provider id when maxConcurrency is 1', async () => {
+    const callOrder: string[] = [];
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judgeOne: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge-one'),
+      callApi: vi.fn(async () => {
+        callOrder.push('judge-one');
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'judge one passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const judgeTwo: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge-two'),
+      callApi: vi.fn(async () => {
+        callOrder.push('judge-two');
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'judge two passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [
+            { type: 'llm-rubric', value: 'Judge alpha one', provider: judgeOne },
+            { type: 'llm-rubric', value: 'Judge alpha two', provider: judgeTwo },
+          ],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [
+            { type: 'llm-rubric', value: 'Judge beta one', provider: judgeOne },
+            { type: 'llm-rubric', value: 'Judge beta two', provider: judgeTwo },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(callOrder).toEqual(['judge-one', 'judge-one', 'judge-two', 'judge-two']);
+  });
+
+  it('keeps model-graded assertions row-first when prompts use _conversation', async () => {
+    const callOrder: string[] = [];
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => {
+        const topic = prompt.endsWith('Current: alpha') ? 'alpha' : 'beta';
+        callOrder.push(`target:${topic}`);
+        return {
+          output: `Target output for ${topic}`,
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => {
+        callOrder.push('judge');
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'judge passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [
+        toPrompt(
+          '{% for turn in _conversation %}{{ turn.input }} => {{ turn.output }}\n{% endfor %}Current: {{topic}}',
+        ),
+      ],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [{ type: 'llm-rubric', value: 'Judge beta', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(callOrder).toEqual(['target:alpha', 'judge', 'target:beta', 'judge']);
+  });
+
+  it('records deferred model-graded provider failures as row errors when maxConcurrency is 1', async () => {
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async (prompt: string) => {
+        if (prompt.includes('Judge alpha')) {
+          throw new Error('grader exploded');
+        }
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'judge passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [{ type: 'llm-rubric', value: 'Judge beta', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    const failedResult = summary.results.find((result) => result.vars.topic === 'alpha');
+    expect(summary.stats.errors).toBe(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.results).toHaveLength(2);
+    expect(failedResult?.failureReason).toBe(ResultFailureReason.ERROR);
+    expect(failedResult?.error).toContain('grader exploded');
+  });
+
+  it('stops grouped serial evals after a non-transient target status', async () => {
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        metadata: {
+          http: {
+            status: 403,
+            statusText: 'Forbidden',
+          },
+        },
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => ({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'judge passed' }),
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [{ type: 'llm-rubric', value: 'Judge beta', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(provider.callApi).toHaveBeenCalledTimes(1);
+    expect(judge.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.results).toHaveLength(1);
+    expect(summary.results[0].vars.topic).toBe('alpha');
+  });
+
+  it('groups model-graded assert-set children by provider id when maxConcurrency is 1', async () => {
+    const callOrder: string[] = [];
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judgeOne: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge-one'),
+      callApi: vi.fn(async () => {
+        callOrder.push('judge-one');
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'judge one passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const judgeTwo: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge-two'),
+      callApi: vi.fn(async () => {
+        callOrder.push('judge-two');
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'judge two passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [
+            {
+              type: 'assert-set',
+              assert: [
+                { type: 'llm-rubric', value: 'Judge alpha one', provider: judgeOne },
+                { type: 'llm-rubric', value: 'Judge alpha two', provider: judgeTwo },
+              ],
+            },
+          ],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [
+            {
+              type: 'assert-set',
+              assert: [
+                { type: 'llm-rubric', value: 'Judge beta one', provider: judgeOne },
+                { type: 'llm-rubric', value: 'Judge beta two', provider: judgeTwo },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(callOrder).toEqual(['judge-one', 'judge-one', 'judge-two', 'judge-two']);
+  });
+
+  it('keeps multi-call model-graded assertions grouped by provider id when maxConcurrency is 1', async () => {
+    const callOrder: string[] = [];
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const createJudge = (id: string): ApiProvider => ({
+      id: vi.fn().mockReturnValue(id),
+      callApi: vi.fn(async (_prompt: string, context?: { prompt?: { label?: string } }) => {
+        const label = context?.prompt?.label ?? 'unknown';
+        callOrder.push(`${id}:${label}`);
+
+        return {
+          output:
+            label === 'g-eval-steps'
+              ? JSON.stringify({ steps: ['Check the answer'] })
+              : JSON.stringify({ score: 10, reason: 'passed' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    });
+    const judgeOne = createJudge('judge-one');
+    const judgeTwo = createJudge('judge-two');
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [
+            { type: 'g-eval', value: 'Judge alpha one', provider: judgeOne },
+            { type: 'g-eval', value: 'Judge alpha two', provider: judgeTwo },
+          ],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [
+            { type: 'g-eval', value: 'Judge beta one', provider: judgeOne },
+            { type: 'g-eval', value: 'Judge beta two', provider: judgeTwo },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(callOrder).toEqual([
+      'judge-one:g-eval-steps',
+      'judge-one:g-eval-steps',
+      'judge-one:g-eval',
+      'judge-one:g-eval',
+      'judge-two:g-eval-steps',
+      'judge-two:g-eval-steps',
+      'judge-two:g-eval',
+      'judge-two:g-eval',
+    ]);
+  });
+
+  it('isolates deferred grading cache entries by repeat index', async () => {
+    await clearCache();
+
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        cached: false,
+        output: 'target output',
+        tokenUsage: createEmptyTokenUsage(),
+      }),
+    };
+
+    let gradingCacheMissCount = 0;
+    const gradingProvider: ApiProvider = {
+      id: () => 'grading-provider',
+      callApi: vi.fn().mockImplementation(async () => {
+        const cache = getCache();
+        const cachedOutput = await cache.get<string>('deferred-grading-key');
+        if (cachedOutput) {
+          return {
+            cached: true,
+            output: cachedOutput,
+            tokenUsage: createEmptyTokenUsage(),
+          };
+        }
+
+        gradingCacheMissCount += 1;
+        const output = JSON.stringify({
+          pass: true,
+          reason: `grader-repeat-${gradingCacheMissCount}`,
+          score: 1,
+        });
+        await cache.set('deferred-grading-key', output);
+        return {
+          cached: false,
+          output,
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [{ type: 'llm-rubric', value: 'Output should pass', provider: gradingProvider }],
+        },
+      ],
+    };
+
+    const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, firstEval, { maxConcurrency: 1, repeat: 2 });
+
+    expect(gradingCacheMissCount).toBe(2);
+
+    const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, secondEval, { maxConcurrency: 1, repeat: 2 });
+
+    expect(gradingCacheMissCount).toBe(2);
+    expect(gradingProvider.callApi).toHaveBeenCalledTimes(4);
+  });
+
+  it('serializes async assert-set judges when provider call queue is active', async () => {
+    // Regression: without forcing assertion concurrency to 1 when the grouping
+    // queue is active, async graders inside an assert-set would interleave
+    // judges across rows (judge-one row1, judge-two row1, judge-one row2, ...)
+    // and defeat the single-judge-drain guarantee.
+    const callOrder: string[] = [];
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const makeAsyncJudge = (id: string): ApiProvider => ({
+      id: vi.fn().mockReturnValue(id),
+      callApi: vi.fn(async () => {
+        // Intentionally yield through both microtask and macrotask queues so
+        // this regression test can expose the cross-row interleaving that
+        // assertion concurrency caused.
+        await Promise.resolve();
+        await new Promise<void>((resolve) => setImmediate(resolve));
+        callOrder.push(id);
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: `${id} passed` }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    });
+    const judgeOne = makeAsyncJudge('judge-one');
+    const judgeTwo = makeAsyncJudge('judge-two');
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [
+            {
+              type: 'assert-set',
+              assert: [
+                { type: 'llm-rubric', value: 'Judge alpha one', provider: judgeOne },
+                { type: 'llm-rubric', value: 'Judge alpha two', provider: judgeTwo },
+              ],
+            },
+          ],
+        },
+        {
+          vars: { topic: 'beta' },
+          assert: [
+            {
+              type: 'assert-set',
+              assert: [
+                { type: 'llm-rubric', value: 'Judge beta one', provider: judgeOne },
+                { type: 'llm-rubric', value: 'Judge beta two', provider: judgeTwo },
+              ],
+            },
+          ],
+        },
+        {
+          vars: { topic: 'gamma' },
+          assert: [
+            {
+              type: 'assert-set',
+              assert: [
+                { type: 'llm-rubric', value: 'Judge gamma one', provider: judgeOne },
+                { type: 'llm-rubric', value: 'Judge gamma two', provider: judgeTwo },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(3);
+    expect(callOrder).toEqual([
+      'judge-one',
+      'judge-one',
+      'judge-one',
+      'judge-two',
+      'judge-two',
+      'judge-two',
+    ]);
+  });
+
+  it('does not log error-level message when deferred grading is aborted', async () => {
+    const { default: logger } = await import('../../src/logger');
+    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);
+
+    const abortController = new AbortController();
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => {
+        abortController.abort();
+        const abortError = new Error('Aborted');
+        abortError.name = 'AbortError';
+        throw abortError;
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {
+      maxConcurrency: 1,
+      abortSignal: abortController.signal,
+    });
+
+    const gradingErrorLogs = errorSpy.mock.calls.filter(
+      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
+    );
+    expect(gradingErrorLogs).toHaveLength(0);
+
+    const failedResult = (await evalRecord.toEvaluateSummary()).results.find(
+      (result) => result.vars.topic === 'alpha',
+    );
+    expect(failedResult?.error).toMatch(/^Aborted: /);
+
+    errorSpy.mockRestore();
+  });
+
+  it('suppresses the error log when deferred grading throws AbortException under abort', async () => {
+    const { default: logger } = await import('../../src/logger');
+    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);
+
+    const abortController = new AbortController();
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => {
+        abortController.abort();
+        const abortException = new Error('Python provider cancelled');
+        abortException.name = 'AbortException';
+        throw abortException;
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {
+      maxConcurrency: 1,
+      abortSignal: abortController.signal,
+    });
+
+    const gradingErrorLogs = errorSpy.mock.calls.filter(
+      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
+    );
+    expect(gradingErrorLogs).toHaveLength(0);
+
+    errorSpy.mockRestore();
+  });
+
+  it('still logs error-level when abort-shaped error fires WITHOUT an aborted signal', async () => {
+    // Regression guard: third-party SDKs sometimes surface unrelated cancellation
+    // as AbortError. If the evaluator's abort signal is NOT tripped, these are
+    // real bugs and must still surface at error level.
+    const { default: logger } = await import('../../src/logger');
+    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);
+
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => {
+        const sdkAbortLookalike = new Error('fetch aborted by keepalive');
+        sdkAbortLookalike.name = 'AbortError';
+        throw sdkAbortLookalike;
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+
+    const gradingErrorLogs = errorSpy.mock.calls.filter(
+      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
+    );
+    expect(gradingErrorLogs.length).toBeGreaterThanOrEqual(1);
+
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary.stats.errors).toBe(1);
+    expect(summary.results[0].failureReason).toBe(ResultFailureReason.ERROR);
+    expect(summary.results[0].error).not.toMatch(/^Aborted: /);
+
+    errorSpy.mockRestore();
+  });
+
+  it('still logs error when a non-abort-shaped error fires during an unrelated abort', async () => {
+    // Regression guard: if abort is in flight but the caught error is a real
+    // bug (SyntaxError, TypeError, etc.), we must not silently suppress it.
+    const { default: logger } = await import('../../src/logger');
+    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);
+
+    const abortController = new AbortController();
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => {
+        abortController.abort();
+        throw new SyntaxError('Unexpected token in grader output');
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {
+      maxConcurrency: 1,
+      abortSignal: abortController.signal,
+    });
+
+    const gradingErrorLogs = errorSpy.mock.calls.filter(
+      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
+    );
+    expect(gradingErrorLogs.length).toBeGreaterThanOrEqual(1);
+
+    errorSpy.mockRestore();
+  });
+
+  it('logs grouping-active info when serial eval contains model-graded assertions', async () => {
+    const { default: logger } = await import('../../src/logger');
+    const infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);
+
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => ({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'ok' }),
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+
+    const groupingLogs = infoSpy.mock.calls.filter(
+      ([message]) =>
+        typeof message === 'string' &&
+        message.includes('Grouping model-graded assertions by provider'),
+    );
+    expect(groupingLogs).toHaveLength(1);
+
+    infoSpy.mockRestore();
+  });
+
+  it('does not log grouping info when there are no model-graded assertions', async () => {
+    const { default: logger } = await import('../../src/logger');
+    const infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);
+
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt {{topic}}')],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'contains', value: 'Target' }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+
+    const groupingLogs = infoSpy.mock.calls.filter(
+      ([message]) =>
+        typeof message === 'string' &&
+        (message.includes('Grouping model-graded assertions by provider') ||
+          message.includes('Serial grading grouping disabled')),
+    );
+    expect(groupingLogs).toHaveLength(0);
+
+    infoSpy.mockRestore();
+  });
+
+  it('logs grouping-disabled reason when conversation var forces per-row ordering', async () => {
+    const { default: logger } = await import('../../src/logger');
+    const infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);
+
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async () => ({
+        output: 'Target output',
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => ({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'ok' }),
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [
+        toPrompt('{% for turn in _conversation %}{{ turn.input }}{% endfor %}Current: {{topic}}'),
+      ],
+      tests: [
+        {
+          vars: { topic: 'alpha' },
+          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
+
+    const disabledLogs = infoSpy.mock.calls.filter(
+      ([message]) =>
+        typeof message === 'string' &&
+        message.includes('Serial grading grouping disabled') &&
+        message.includes('conversation variables'),
+    );
+    expect(disabledLogs).toHaveLength(1);
+
+    infoSpy.mockRestore();
+  });
+
+  it('keeps parallel assertion dispatch when the grouping queue is NOT active', async () => {
+    // Regression guard for the `? 1 : ASSERTIONS_MAX_CONCURRENCY` ternary in
+    // runAssertions. Without the queue present (non-deferred concurrent eval),
+    // per-test assertions must still fan out so we don't silently 3x-throttle
+    // normal eval users.
+    let inFlight = 0;
+    let maxInFlight = 0;
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('target-provider'),
+      callApi: vi.fn(async (prompt: string) => ({
+        output: `Target output for ${prompt}`,
+        tokenUsage: createEmptyTokenUsage(),
+      })),
+    };
+    // Concurrency barrier: hold every in-flight judge until at least two have
+    // started, then release all of them together. This makes maxInFlight
+    // observation independent of wall-clock pacing.
+    let releaseHold: (() => void) | undefined;
+    const holdPromise = new Promise<void>((resolve) => {
+      releaseHold = resolve;
+    });
+    const judge: ApiProvider = {
+      id: vi.fn().mockReturnValue('judge'),
+      callApi: vi.fn(async () => {
+        inFlight += 1;
+        maxInFlight = Math.max(maxInFlight, inFlight);
+        if (inFlight >= 2) {
+          releaseHold?.();
+        }
+        let resolveFallback!: () => void;
+        const fallback = new Promise<void>((r) => {
+          resolveFallback = r;
+        });
+        const fallbackHandle = setTimeout(() => resolveFallback(), 100);
+        try {
+          await Promise.race([holdPromise, fallback]);
+        } finally {
+          clearTimeout(fallbackHandle);
+        }
+        inFlight -= 1;
+        return {
+          output: JSON.stringify({ pass: true, score: 1, reason: 'ok' }),
+          tokenUsage: createEmptyTokenUsage(),
+        };
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            { type: 'llm-rubric', value: 'Judge one', provider: judge },
+            { type: 'llm-rubric', value: 'Judge two', provider: judge },
+            { type: 'llm-rubric', value: 'Judge three', provider: judge },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    // maxConcurrency > 1 takes the non-grouped path where the queue is absent.
+    await evaluate(testSuite, evalRecord, { maxConcurrency: 5 });
+
+    expect(inFlight).toBe(0);
+    expect(maxInFlight).toBeGreaterThanOrEqual(2);
+  });
+});

test/evaluator/helpers.ts

@@ -0,0 +1,84 @@
+import { vi } from 'vitest';
+
+import type { ApiProvider, Prompt, ProviderResponse } from '../../src/types/index';
+
+const mockApiProvider: ApiProvider = {
+  id: vi.fn<ApiProvider['id']>(),
+  callApi: vi.fn<ApiProvider['callApi']>(),
+};
+
+const mockApiProvider2: ApiProvider = {
+  id: vi.fn<ApiProvider['id']>(),
+  callApi: vi.fn<ApiProvider['callApi']>(),
+};
+
+const mockReasoningApiProvider: ApiProvider = {
+  id: vi.fn<ApiProvider['id']>(),
+  callApi: vi.fn<ApiProvider['callApi']>(),
+};
+
+const mockGradingApiProviderPasses: ApiProvider = {
+  id: vi.fn<ApiProvider['id']>(),
+  callApi: vi.fn<ApiProvider['callApi']>(),
+};
+
+const mockGradingApiProviderFails: ApiProvider = {
+  id: vi.fn<ApiProvider['id']>(),
+  callApi: vi.fn<ApiProvider['callApi']>(),
+};
+
+function defaultTokenUsage() {
+  return { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 };
+}
+
+function defaultProviderResponse(): ProviderResponse {
+  return {
+    output: 'Test output',
+    tokenUsage: defaultTokenUsage(),
+  };
+}
+
+function resetProvider(provider: ApiProvider, id: string, response: ProviderResponse) {
+  vi.mocked(provider.id).mockReset().mockReturnValue(id);
+  vi.mocked(provider.callApi).mockReset().mockResolvedValue(response);
+}
+
+function resetMockProviders() {
+  resetProvider(mockApiProvider, 'test-provider', defaultProviderResponse());
+  resetProvider(mockApiProvider2, 'test-provider-2', defaultProviderResponse());
+  resetProvider(mockReasoningApiProvider, 'test-reasoning-provider', {
+    output: 'Test output',
+    tokenUsage: {
+      total: 21,
+      prompt: 9,
+      completion: 12,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: { reasoning: 11, acceptedPrediction: 12, rejectedPrediction: 13 },
+    },
+  });
+  resetProvider(mockGradingApiProviderPasses, 'test-grading-provider', {
+    output: JSON.stringify({ pass: true, reason: 'Test grading output' }),
+    tokenUsage: defaultTokenUsage(),
+  });
+  resetProvider(mockGradingApiProviderFails, 'test-grading-provider', {
+    output: JSON.stringify({ pass: false, reason: 'Grading failed reason' }),
+    tokenUsage: defaultTokenUsage(),
+  });
+}
+
+function toPrompt(text: string): Prompt {
+  return { raw: text, label: text };
+}
+
+resetMockProviders();
+
+export {
+  mockApiProvider,
+  mockApiProvider2,
+  mockGradingApiProviderFails,
+  mockGradingApiProviderPasses,
+  mockReasoningApiProvider,
+  resetMockProviders,
+  toPrompt,
+};

test/evaluator/inMemoryStore.test.ts

@@ -0,0 +1,177 @@
+import { describe, expect, it } from 'vitest';
+import {
+  type InMemoryEvaluation,
+  InMemoryEvaluationStore,
+} from '../../src/evaluator/inMemoryStore';
+import { ResultFailureReason } from '../../src/types/index';
+import { createCompletedPrompt, createEvaluateResult } from '../factories/eval';
+
+function createEvaluation(overrides: Partial<InMemoryEvaluation> = {}): InMemoryEvaluation {
+  return {
+    id: 'in-memory-eval',
+    config: { description: 'In-memory evaluation' },
+    persisted: false,
+    prompts: [],
+    results: [],
+    vars: [],
+    resultPersistenceFailed: false,
+    finalResults: [],
+    failedResults: [],
+    ...overrides,
+  };
+}
+
+describe('InMemoryEvaluationStore', () => {
+  it('exposes the evaluation record and its core properties', () => {
+    const evaluation = createEvaluation({
+      persisted: true,
+      prompts: [createCompletedPrompt('Initial prompt')],
+      results: [createEvaluateResult()],
+    });
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    expect(store.evaluation).toBe(evaluation);
+    expect(store.id).toBe(evaluation.id);
+    expect(store.config).toBe(evaluation.config);
+    expect(store.persisted).toBe(true);
+    expect(store.prompts).toBe(evaluation.prompts);
+    expect(store.results).toBe(evaluation.results);
+    expect(store.resultPersistenceFailed).toBe(false);
+  });
+
+  it('appends results, replaces prompts, and reads results by test index', async () => {
+    const evaluation = createEvaluation();
+    const store = new InMemoryEvaluationStore(evaluation);
+    const first = createEvaluateResult({ testIdx: 1, promptIdx: 0 });
+    const second = createEvaluateResult({ testIdx: 2, promptIdx: 0 });
+    const third = createEvaluateResult({ testIdx: 1, promptIdx: 1 });
+    const prompts = [createCompletedPrompt('First'), createCompletedPrompt('Second')];
+
+    await store.appendResult(first);
+    await store.appendResult(second);
+    await store.appendResult(third);
+    await store.appendPrompts(prompts);
+
+    expect(await store.readResults()).toBe(evaluation.results);
+    expect(await store.readResults()).toEqual([first, second, third]);
+    expect(await store.readResultsByTestIdx(1)).toEqual([first, third]);
+    expect(evaluation.prompts).toBe(prompts);
+  });
+
+  it('uses deterministic testIdx:promptIdx pairs for resume filtering', async () => {
+    const evaluation = createEvaluation({
+      results: [
+        createEvaluateResult({
+          testIdx: 3,
+          promptIdx: 4,
+          failureReason: ResultFailureReason.NONE,
+        }),
+        createEvaluateResult({
+          testIdx: 3,
+          promptIdx: 5,
+          failureReason: ResultFailureReason.ERROR,
+        }),
+        createEvaluateResult({
+          testIdx: 3,
+          promptIdx: 4,
+          failureReason: ResultFailureReason.ASSERT,
+        }),
+      ],
+    });
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    expect(await store.readCompletedIndexPairs()).toEqual(new Set(['3:4', '3:5']));
+    expect(await store.readCompletedIndexPairs({ excludeErrors: false })).toEqual(
+      new Set(['3:4', '3:5']),
+    );
+    expect(await store.readCompletedIndexPairs({ excludeErrors: true })).toEqual(new Set(['3:4']));
+  });
+
+  it('records final results with last-write-wins index semantics', () => {
+    const initial = createEvaluateResult({ testIdx: 0, promptIdx: 1, score: 0 });
+    const replacement = createEvaluateResult({ testIdx: 0, promptIdx: 1, score: 1 });
+    const other = createEvaluateResult({ testIdx: 1, promptIdx: 0 });
+    const evaluation = createEvaluation({ finalResults: [initial] });
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    store.recordFinalResult(other);
+    store.recordFinalResult(replacement);
+
+    expect(evaluation.finalResults).toEqual([replacement, other]);
+  });
+
+  it('tracks persistence failures by index and preserves result mutations across reads', async () => {
+    const first = createEvaluateResult({ testIdx: 2, promptIdx: 0, score: 1 });
+    const otherTest = createEvaluateResult({ testIdx: 3, promptIdx: 0 });
+    const evaluation = createEvaluation();
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    store.recordResultPersistenceFailure(first);
+    store.recordResultPersistenceFailure(otherTest);
+
+    expect(store.resultPersistenceFailed).toBe(true);
+    expect(store.hasResultPersistenceFailure({ testIdx: 2, promptIdx: 0 })).toBe(true);
+    expect(store.hasResultPersistenceFailure({ testIdx: 2, promptIdx: 1 })).toBe(false);
+
+    const [failed] = await store.readFailedResultsByTestIdx(2);
+    failed.score = 0;
+    expect((await store.readFailedResultsByTestIdx(2))[0]).toBe(failed);
+    expect((await store.readFailedResultsByTestIdx(2))[0].score).toBe(0);
+  });
+
+  it('replaces a re-recorded persistence failure for the same index', async () => {
+    const first = createEvaluateResult({ testIdx: 2, promptIdx: 0, score: 0 });
+    const replacement = createEvaluateResult({ testIdx: 2, promptIdx: 0, score: 1 });
+    const evaluation = createEvaluation({ failedResults: [first] });
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    expect(store.resultPersistenceFailed).toBe(true);
+    store.recordResultPersistenceFailure(replacement);
+
+    expect(await store.readFailedResultsByTestIdx(2)).toEqual([replacement]);
+    expect(evaluation.failedResults).toEqual([replacement]);
+  });
+
+  it('saves the evaluation and upserts saved results by index', async () => {
+    const original = createEvaluateResult({ testIdx: 0, promptIdx: 0, score: 0 });
+    const replacement = createEvaluateResult({ testIdx: 0, promptIdx: 0, score: 1 });
+    const additional = createEvaluateResult({ testIdx: 0, promptIdx: 1 });
+    const evaluation = createEvaluation({ results: [original] });
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    await store.saveResult(replacement);
+    await store.saveResult(additional);
+    await store.save();
+
+    expect(evaluation.persisted).toBe(true);
+    expect(evaluation.results).toEqual([replacement, additional]);
+  });
+
+  it('updates vars and valid durations while retaining generation duration', () => {
+    const evaluation = createEvaluation({
+      generationDurationMs: 25,
+      durationMs: 25,
+    });
+    const store = new InMemoryEvaluationStore(evaluation);
+
+    store.setVars(['topic', 'language']);
+    store.setDurationMs(75);
+
+    expect(evaluation.vars).toEqual(['topic', 'language']);
+    expect(evaluation.evaluationDurationMs).toBe(75);
+    expect(evaluation.durationMs).toBe(100);
+
+    store.setDurationMs(-1);
+    store.setDurationMs(Number.NaN);
+    store.setDurationMs(Number.POSITIVE_INFINITY);
+    expect(evaluation.evaluationDurationMs).toBe(75);
+    expect(evaluation.durationMs).toBe(100);
+  });
+
+  it('returns EvaluateResult values unchanged', () => {
+    const store = new InMemoryEvaluationStore(createEvaluation());
+    const result = createEvaluateResult();
+
+    expect(store.toEvaluateResult(result)).toBe(result);
+  });
+});

test/evaluator/lifecycle.ts

@@ -0,0 +1,49 @@
+import { afterAll, afterEach, beforeAll, beforeEach, describe, vi } from 'vitest';
+import { clearCache } from '../../src/cache';
+import cliState from '../../src/cliState';
+import { runExtensionHook } from '../../src/evaluatorHelpers';
+import { runDbMigrations } from '../../src/migrate';
+import { resetMockProviders } from './helpers';
+
+export function describeEvaluator(name: string, defineTests: () => void) {
+  describe(name, () => {
+    beforeAll(async () => {
+      await runDbMigrations();
+    });
+
+    beforeEach(() => {
+      vi.useRealTimers();
+      vi.clearAllMocks();
+      resetMockProviders();
+      vi.mocked(runExtensionHook).mockReset();
+      vi.mocked(runExtensionHook).mockImplementation(
+        async (_extensions, _hookName, context) => context,
+      );
+      cliState.resume = false;
+      cliState.retryMode = false;
+      cliState.basePath = '';
+      cliState.webUI = false;
+    });
+
+    afterEach(async () => {
+      vi.useRealTimers();
+      resetMockProviders();
+      vi.clearAllMocks();
+      cliState.resume = false;
+      cliState.retryMode = false;
+      cliState.basePath = '';
+      cliState.webUI = false;
+      await clearCache();
+      if (global.gc) {
+        global.gc();
+      }
+    });
+
+    afterAll(() => {
+      vi.restoreAllMocks();
+      vi.resetModules();
+    });
+
+    defineTests();
+  });
+}

test/evaluator/metadata.test.ts

@@ -0,0 +1,579 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { expect, it, vi } from 'vitest';
+import { FILE_METADATA_KEY } from '../../src/constants';
+import { evaluate } from '../../src/evaluator';
+import { runExtensionHook } from '../../src/evaluatorHelpers';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type EvaluateSummaryV3, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator metadata', () => {
+  it('evaluate with metadata passed to test transform', async () => {
+    const mockApiProviderWithMetadata: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-metadata'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        metadata: { responseTime: 123, modelVersion: 'v1.0' },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithMetadata],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Output: Test output, Metadata: {"responseTime":123,"modelVersion":"v1.0"}',
+            },
+          ],
+          options: {
+            transform:
+              'context?.metadata ? `Output: ${output}, Metadata: ${JSON.stringify(context.metadata)}` : output',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProviderWithMetadata.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe(
+      'Output: Test output, Metadata: {"responseTime":123,"modelVersion":"v1.0"}',
+    );
+  });
+
+  it('evaluate with metadata passed to test transform - no metadata case', async () => {
+    const mockApiProviderNoMetadata: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-no-metadata'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderNoMetadata],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'No metadata: Test output',
+            },
+          ],
+          options: {
+            transform: 'context?.metadata ? `Has metadata: ${output}` : `No metadata: ${output}`',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProviderNoMetadata.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('No metadata: Test output');
+  });
+
+  it('evaluate with metadata passed to test transform - empty metadata', async () => {
+    const mockApiProviderEmptyMetadata: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-empty-metadata'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        metadata: {},
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderEmptyMetadata],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Empty metadata: Test output',
+            },
+          ],
+          options: {
+            transform:
+              '(context?.metadata && Object.keys(context.metadata).length === 0) ? `Empty metadata: ${output}` : output',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProviderEmptyMetadata.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Empty metadata: Test output');
+  });
+
+  it('evaluate with metadata preserved alongside other context properties', async () => {
+    const mockApiProviderWithMetadata: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-metadata-context'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        metadata: { modelInfo: 'gpt-4' },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithMetadata],
+      prompts: [toPrompt('Test {{ var }}')],
+      tests: [
+        {
+          vars: { var: 'value' },
+          assert: [
+            {
+              type: 'equals',
+              value: 'All context: Test output',
+            },
+          ],
+          options: {
+            transform:
+              '(Boolean(context?.vars) && Boolean(context?.prompt) && Boolean(context?.metadata)) ? `All context: ${output}` : `Missing context: ${output}`',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProviderWithMetadata.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('All context: Test output');
+  });
+
+  it('merges metadata correctly for regular tests', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      defaultTest: {
+        metadata: { defaultKey: 'defaultValue' },
+      },
+      tests: [
+        {
+          metadata: { testKey: 'testValue' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const results = await evalRecord.getResults();
+    expect(results[0].testCase.metadata).toEqual({
+      defaultKey: 'defaultValue',
+      testKey: 'testValue',
+    });
+  });
+
+  it('merges response metadata with test metadata', async () => {
+    const mockProviderWithMetadata: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-with-metadata'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        metadata: { responseKey: 'responseValue' },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockProviderWithMetadata],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          metadata: { testKey: 'testValue' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const results = await evalRecord.getResults();
+
+    // Check that both test metadata and response metadata are present in the result
+    expect(results[0].metadata).toEqual({
+      testKey: 'testValue',
+      responseKey: 'responseValue',
+      [FILE_METADATA_KEY]: {},
+    });
+  });
+
+  it('evaluate with _conversation variable', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockImplementation((prompt) =>
+        Promise.resolve({
+          output: prompt,
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        }),
+      ),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('{{ var1 }} {{ _conversation[0].output }}')],
+      tests: [
+        {
+          vars: { var1: 'First run' },
+        },
+        {
+          vars: { var1: 'Second run' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.stats.successes).toBe(2);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('First run ');
+    expect(summary.results[1].response?.output).toBe('Second run First run ');
+  });
+
+  it('should maintain separate conversation histories based on metadata.conversationId', async () => {
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockImplementation((_prompt) => ({
+        output: 'Test output',
+      })),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [
+        {
+          raw: '{% for completion in _conversation %}User: {{ completion.input }}\nAssistant: {{ completion.output }}\n{% endfor %}User: {{ question }}',
+          label: 'Conversation test',
+        },
+      ],
+      tests: [
+        // First conversation
+        {
+          vars: { question: 'Question 1A' },
+          metadata: { conversationId: 'conversation1' },
+        },
+        {
+          vars: { question: 'Question 1B' },
+          metadata: { conversationId: 'conversation1' },
+        },
+        // Second conversation
+        {
+          vars: { question: 'Question 2A' },
+          metadata: { conversationId: 'conversation2' },
+        },
+        {
+          vars: { question: 'Question 2B' },
+          metadata: { conversationId: 'conversation2' },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    // Check that the API was called with the correct prompts
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+
+    // First conversation, first question
+    expect(mockApiProvider.callApi).toHaveBeenNthCalledWith(
+      1,
+      expect.stringContaining('User: Question 1A'),
+      expect.anything(),
+      undefined,
+    );
+
+    // First conversation, second question (should include history)
+    expect(mockApiProvider.callApi).toHaveBeenNthCalledWith(
+      2,
+      expect.stringContaining('User: Question 1A\nAssistant: Test output\nUser: Question 1B'),
+      expect.anything(),
+      undefined,
+    );
+
+    // Second conversation, first question (should NOT include first conversation)
+    expect(mockApiProvider.callApi).toHaveBeenNthCalledWith(
+      3,
+      expect.stringContaining('User: Question 2A'),
+      expect.anything(),
+      undefined,
+    );
+
+    // Second conversation, second question (should only include second conversation history)
+    expect(mockApiProvider.callApi).toHaveBeenNthCalledWith(
+      4,
+      expect.stringContaining('User: Question 2A\nAssistant: Test output\nUser: Question 2B'),
+      expect.anything(),
+      undefined,
+    );
+  });
+
+  it('should include sessionId in metadata for afterEach hook', async () => {
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        sessionId: 'test-session-123',
+      }),
+    };
+
+    const mockExtension = 'file://test-extension.js';
+    let capturedContext: any;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'afterEach') {
+        capturedContext = context;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { var1: 'value1' },
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedContext).toBeDefined();
+    expect(capturedContext.result.metadata.sessionId).toBe('test-session-123');
+  });
+
+  it('should include sessionIds array from test metadata for iterative providers', async () => {
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+      }),
+    };
+
+    const mockExtension = 'file://test-extension.js';
+    let capturedContext: any;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'afterEach') {
+        capturedContext = context;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { var1: 'value1' },
+          metadata: {
+            sessionIds: ['iter-session-1', 'iter-session-2', 'iter-session-3'],
+          },
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedContext).toBeDefined();
+    expect(capturedContext.result.metadata.sessionIds).toEqual([
+      'iter-session-1',
+      'iter-session-2',
+      'iter-session-3',
+    ]);
+    expect(capturedContext.result.metadata.sessionId).toBeUndefined();
+  });
+
+  it('should persist afterEach hook namedScores, metadata, and response.metadata into result and metrics', async () => {
+    const mockExtension = 'file://test-extension.js:afterEach';
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'afterEach') {
+        const ctx = context as { test: any; result: any };
+        return {
+          ...ctx,
+          result: {
+            ...ctx.result,
+            namedScores: {
+              ...ctx.result.namedScores,
+              hook_metric: 42,
+            },
+            metadata: {
+              ...ctx.result.metadata,
+              hook_key: 'hook_value',
+            },
+            response: {
+              ...ctx.result.response,
+              metadata: {
+                ...ctx.result.response?.metadata,
+                session_url: 'https://example.com/session/123',
+              },
+            },
+          },
+        };
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: {},
+          assert: [{ type: 'equals', value: 'Test output' }],
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = (await evalRecord.toEvaluateSummary()) as EvaluateSummaryV3;
+
+    // Verify hook's namedScores flowed into prompt metrics
+    expect(summary.prompts[0].metrics?.namedScores).toHaveProperty('hook_metric', 42);
+
+    // Verify hook's metadata and namedScores are in the persisted result
+    const result = summary.results[0];
+    expect(result.metadata).toHaveProperty('hook_key', 'hook_value');
+    expect(result.namedScores).toHaveProperty('hook_metric', 42);
+    expect(result.response?.metadata).toHaveProperty(
+      'session_url',
+      'https://example.com/session/123',
+    );
+  });
+
+  it('should persist row without hook modifications when afterEach hook throws', async () => {
+    const mockExtension = 'file://test-extension.js:afterEach';
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, _context) => {
+      if (hookName === 'afterEach') {
+        throw new Error('Hook exploded');
+      }
+      return _context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: {},
+          assert: [{ type: 'equals', value: 'Test output' }],
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = (await evalRecord.toEvaluateSummary()) as EvaluateSummaryV3;
+
+    // Row should still be persisted despite hook failure
+    expect(summary.results).toHaveLength(1);
+    expect(summary.results[0].success).toBe(true);
+    expect(summary.stats.successes).toBe(1);
+  });
+
+  it('drops stale provider headers when afterEach replaces response metadata', async () => {
+    const outputPath = path.join(os.tmpdir(), `promptfoo-evaluator-${randomUUID()}.jsonl`);
+    const mockExtension = 'file://test-extension.js:afterEach';
+    const provider: ApiProvider = {
+      id: vi.fn().mockReturnValue('metadata-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        metadata: {
+          headers: {
+            authorization: 'Bearer provider-secret',
+            'x-safe-debug': 'provider-debug',
+          },
+        },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    vi.mocked(runExtensionHook).mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName !== 'afterEach') {
+        return context;
+      }
+      const ctx = context as { test: any; result: any };
+      return {
+        ...ctx,
+        result: {
+          ...ctx.result,
+          response: {
+            ...ctx.result.response,
+            metadata: {
+              hook_key: 'hook_value',
+            },
+          },
+        },
+      };
+    });
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+      extensions: [mockExtension],
+    };
+
+    try {
+      const evalRecord = await Eval.create({ outputPath }, testSuite.prompts, {
+        id: randomUUID(),
+      });
+      await evaluate(testSuite, evalRecord, {});
+      const summary = (await evalRecord.toEvaluateSummary()) as EvaluateSummaryV3;
+      const [result] = summary.results;
+      const [artifactResult] = fs
+        .readFileSync(outputPath, 'utf8')
+        .split(/\r?\n/)
+        .filter(Boolean)
+        .map((line) => JSON.parse(line));
+
+      // The hook replaced response.metadata, so the legacy top-level metadata.headers copied
+      // from the original transport is stale and must be dropped (not persisted).
+      expect(result.metadata?.headers).toBeUndefined();
+      expect(result.response?.metadata).toEqual({ hook_key: 'hook_value' });
+      expect(artifactResult.metadata.headers).toBeUndefined();
+      expect(artifactResult.response.metadata).toEqual({ hook_key: 'hook_value' });
+      expect(JSON.stringify({ artifactResult, result })).not.toContain('provider-secret');
+    } finally {
+      fs.rmSync(outputPath, { force: true });
+    }
+  });
+});

test/evaluator/metrics.test.ts

@@ -0,0 +1,496 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, ResultFailureReason, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator metrics and scoring', () => {
+  it('evaluator should count named score assertions per metric', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt for namedScoresCount')],
+      tests: [
+        {
+          vars: { var1: 'value1' },
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+              metric: 'Accuracy',
+            },
+            {
+              type: 'contains',
+              value: 'Test',
+              metric: 'Accuracy',
+            },
+            {
+              type: 'javascript',
+              value: 'output.length > 0',
+              metric: 'Completeness',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.results).toHaveLength(1);
+    const result = summary.results[0];
+
+    // Use toMatchObject pattern to avoid conditional expects
+    expect(evalRecord.prompts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          provider: result.provider.id,
+          metrics: expect.objectContaining({
+            namedScoresCount: expect.objectContaining({
+              Accuracy: 2,
+              Completeness: 1,
+            }),
+          }),
+        }),
+      ]),
+    );
+  });
+
+  it('evaluator should count named scores with template metric variables per assertion', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt for template metrics')],
+      tests: [
+        {
+          vars: { metricCategory: 'Accuracy' },
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+              metric: '{{metricCategory}}',
+            },
+            {
+              type: 'contains',
+              value: 'Test',
+              metric: '{{metricCategory}}',
+            },
+          ],
+        },
+        {
+          vars: { metricCategory: 'Accuracy' },
+          assert: [
+            {
+              type: 'javascript',
+              value: 'output.length > 0',
+              metric: '{{metricCategory}}',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(evalRecord.prompts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          metrics: expect.objectContaining({
+            namedScores: expect.objectContaining({
+              Accuracy: expect.any(Number),
+            }),
+            namedScoresCount: expect.objectContaining({
+              Accuracy: 3, // 2 assertions in the first test + 1 in the second
+            }),
+          }),
+        }),
+      ]),
+    );
+  });
+
+  it('evaluator should preserve weighted named score totals alongside prompt denominators', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt for weighted metrics')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+              metric: 'Accuracy',
+              weight: 3,
+            },
+            {
+              type: 'contains',
+              value: 'Missing output',
+              metric: 'Accuracy',
+              weight: 1,
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const results = await evalRecord.getResults();
+    const promptMetrics = evalRecord.prompts[0]?.metrics;
+
+    expect(results[0].namedScores?.Accuracy).toBeCloseTo(0.75, 10);
+    expect(results[0].gradingResult?.namedScoreWeights?.Accuracy).toBe(4);
+    expect(promptMetrics?.namedScores.Accuracy).toBeCloseTo(3, 10);
+    expect(promptMetrics?.namedScoresCount.Accuracy).toBe(2);
+    expect(promptMetrics?.namedScoreWeights?.Accuracy).toBe(4);
+    expect(
+      (promptMetrics?.namedScores.Accuracy ?? 0) /
+        (promptMetrics?.namedScoreWeights?.Accuracy ?? 1),
+    ).toBeCloseTo(0.75, 10);
+  });
+
+  it('evaluator should handle mixed static and template metrics correctly', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test mixed metrics')],
+      tests: [
+        {
+          vars: { category: 'Dynamic' },
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+              metric: '{{category}}',
+            },
+            {
+              type: 'contains',
+              value: 'Test',
+              metric: 'Static',
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(evalRecord.prompts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          metrics: expect.objectContaining({
+            namedScoresCount: expect.objectContaining({
+              Dynamic: 1,
+              Static: 1,
+            }),
+          }),
+        }),
+      ]),
+    );
+  });
+
+  it('evaluator should calculate derived metrics with __count variable for averages', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt for derived metrics')],
+      tests: [
+        {
+          vars: { errorValue: 0.1 },
+          assert: [
+            {
+              type: 'javascript',
+              value: 'context.vars.errorValue',
+              metric: 'APE',
+            },
+          ],
+        },
+        {
+          vars: { errorValue: 0.2 },
+          assert: [
+            {
+              type: 'javascript',
+              value: 'context.vars.errorValue',
+              metric: 'APE',
+            },
+          ],
+        },
+        {
+          vars: { errorValue: 0.3 },
+          assert: [
+            {
+              type: 'javascript',
+              value: 'context.vars.errorValue',
+              metric: 'APE',
+            },
+          ],
+        },
+      ],
+      derivedMetrics: [
+        {
+          name: 'APE_sum',
+          value: 'APE', // Should be 0.1 + 0.2 + 0.3 = 0.6
+        },
+        {
+          name: 'MAPE',
+          value: 'APE / __count', // Should be 0.6 / 3 = 0.2
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    const metrics = evalRecord.prompts[0]?.metrics;
+    expect(metrics).toBeDefined();
+    expect(metrics!.namedScores.APE).toBeCloseTo(0.6, 10);
+    expect(metrics!.namedScores.APE_sum).toBeCloseTo(0.6, 10);
+    expect(metrics!.namedScores.MAPE).toBeCloseTo(0.2, 10);
+  });
+
+  it('evaluator should pass __count to JavaScript function derived metrics', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt for function derived metrics')],
+      tests: [
+        {
+          vars: { score: 10 },
+          assert: [{ type: 'javascript', value: 'context.vars.score', metric: 'Score' }],
+        },
+        {
+          vars: { score: 20 },
+          assert: [{ type: 'javascript', value: 'context.vars.score', metric: 'Score' }],
+        },
+      ],
+      derivedMetrics: [
+        {
+          name: 'AverageScore',
+          value: (namedScores: Record<string, number>) => {
+            // __count should be available in namedScores
+            const count = namedScores.__count || 1;
+            return namedScores.Score / count;
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(evalRecord.prompts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          metrics: expect.objectContaining({
+            namedScores: expect.objectContaining({
+              Score: 30, // 10 + 20
+              AverageScore: 15, // 30 / 2
+            }),
+          }),
+        }),
+      ]),
+    );
+  });
+
+  it('evaluator should calculate __count per-prompt with multiple prompts', async () => {
+    // With 2 prompts and 3 tests, each prompt should have __count = 3 (not 6)
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('First prompt'), toPrompt('Second prompt')],
+      tests: [
+        {
+          vars: { errorValue: 0.1 },
+          assert: [{ type: 'javascript', value: 'context.vars.errorValue', metric: 'APE' }],
+        },
+        {
+          vars: { errorValue: 0.2 },
+          assert: [{ type: 'javascript', value: 'context.vars.errorValue', metric: 'APE' }],
+        },
+        {
+          vars: { errorValue: 0.3 },
+          assert: [{ type: 'javascript', value: 'context.vars.errorValue', metric: 'APE' }],
+        },
+      ],
+      derivedMetrics: [
+        {
+          name: 'MAPE',
+          value: 'APE / __count', // Should be 0.6 / 3 = 0.2 for each prompt
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    // Both prompts should have the same MAPE since they each get 3 test evaluations
+    const metrics0 = evalRecord.prompts[0]?.metrics;
+    const metrics1 = evalRecord.prompts[1]?.metrics;
+    expect(metrics0).toBeDefined();
+    expect(metrics1).toBeDefined();
+    expect(metrics0!.namedScores.APE).toBeCloseTo(0.6, 10);
+    expect(metrics0!.namedScores.MAPE).toBeCloseTo(0.2, 10); // 0.6 / 3, not 0.6 / 6
+    expect(metrics1!.namedScores.APE).toBeCloseTo(0.6, 10);
+    expect(metrics1!.namedScores.MAPE).toBeCloseTo(0.2, 10); // 0.6 / 3, not 0.6 / 6
+  });
+
+  it('evaluator should calculate __count per-prompt with multiple providers', async () => {
+    // With 1 prompt and 2 providers, there are 2 prompt entries (one per provider).
+    // Each prompt entry gets 2 test evaluations, so __count = 2 for each.
+    const mockProvider1: ApiProvider = {
+      id: () => 'provider1',
+      callApi: async () => ({ output: 'response1' }),
+    };
+    const mockProvider2: ApiProvider = {
+      id: () => 'provider2',
+      callApi: async () => ({ output: 'response2' }),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockProvider1, mockProvider2],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { errorValue: 0.1 },
+          assert: [{ type: 'javascript', value: 'context.vars.errorValue', metric: 'APE' }],
+        },
+        {
+          vars: { errorValue: 0.2 },
+          assert: [{ type: 'javascript', value: 'context.vars.errorValue', metric: 'APE' }],
+        },
+      ],
+      derivedMetrics: [
+        {
+          name: 'MAPE',
+          value: 'APE / __count', // 0.3 / 2 = 0.15 for each provider's prompt
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    // With 2 providers, there are 2 prompt entries (one per provider)
+    expect(evalRecord.prompts).toHaveLength(2);
+
+    // Each prompt entry has its own metrics from its 2 test evaluations
+    const metrics0 = evalRecord.prompts[0]?.metrics;
+    const metrics1 = evalRecord.prompts[1]?.metrics;
+    expect(metrics0).toBeDefined();
+    expect(metrics1).toBeDefined();
+    // Each provider's prompt gets APE = 0.3 (0.1 + 0.2)
+    expect(metrics0!.namedScores.APE).toBeCloseTo(0.3, 10);
+    expect(metrics1!.namedScores.APE).toBeCloseTo(0.3, 10);
+    // __count = 2 (tests per provider), so MAPE = 0.3 / 2 = 0.15
+    expect(metrics0!.namedScores.MAPE).toBeCloseTo(0.15, 10);
+    expect(metrics1!.namedScores.MAPE).toBeCloseTo(0.15, 10);
+  });
+
+  it('should apply max-score to overall pass/fail and stats', async () => {
+    const maxScoreProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('max-score-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'hello world',
+        tokenUsage: { total: 1, prompt: 1, completion: 0, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [maxScoreProvider],
+      prompts: [toPrompt('Prompt A'), toPrompt('Prompt B')],
+      tests: [
+        {
+          assert: [{ type: 'contains', value: 'hello' }, { type: 'max-score' }],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+    const results = summary.results
+      .filter((result) => result.testIdx === 0)
+      .sort((a, b) => a.promptIdx - b.promptIdx);
+
+    expect(results).toHaveLength(2);
+    expect(results[0].success).toBe(true);
+    expect(results[1].success).toBe(false);
+    expect(results[1].failureReason).toBe(ResultFailureReason.ASSERT);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(1);
+    expect(summary.stats.errors).toBe(0);
+  });
+
+  it('should apply select-best to overall pass/fail and stats', async () => {
+    // Mock matchesSelectBest to return deterministic results (first wins, second loses)
+    const matchers = await import('../../src/matchers/comparison');
+    const matchesSelectBestSpy = vi.spyOn(matchers, 'matchesSelectBest').mockResolvedValue([
+      { pass: true, score: 1, reason: 'Selected as best' },
+      { pass: false, score: 0, reason: 'Not selected' },
+    ]);
+
+    try {
+      const selectBestProvider: ApiProvider = {
+        id: vi.fn().mockReturnValue('select-best-provider'),
+        callApi: vi.fn().mockResolvedValue({
+          output: 'hello world',
+          tokenUsage: { total: 1, prompt: 1, completion: 0, cached: 0, numRequests: 1 },
+        }),
+      };
+
+      const testSuite: TestSuite = {
+        providers: [selectBestProvider],
+        prompts: [toPrompt('Prompt A'), toPrompt('Prompt B')],
+        tests: [
+          {
+            assert: [
+              { type: 'contains', value: 'hello' },
+              { type: 'select-best', value: 'choose the best one' },
+            ],
+          },
+        ],
+      };
+
+      const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+      await evaluate(testSuite, evalRecord, {});
+      const summary = await evalRecord.toEvaluateSummary();
+      const results = summary.results
+        .filter((result) => result.testIdx === 0)
+        .sort((a, b) => a.promptIdx - b.promptIdx);
+
+      expect(results).toHaveLength(2);
+      expect(results[0].success).toBe(true);
+      expect(results[1].success).toBe(false);
+      expect(results[1].failureReason).toBe(ResultFailureReason.ASSERT);
+      expect(summary.stats.successes).toBe(1);
+      expect(summary.stats.failures).toBe(1);
+      expect(summary.stats.errors).toBe(0);
+    } finally {
+      matchesSelectBestSpy.mockRestore();
+    }
+  });
+
+  it('evaluate with assertScoringFunction', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assertScoringFunction: 'file://path/to/scoring.js:customScore',
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output',
+              metric: 'accuracy',
+            },
+            {
+              type: 'contains',
+              value: 'output',
+              metric: 'relevance',
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].score).toBe(0.75);
+  });
+});

test/evaluator/options.test.ts

@@ -0,0 +1,265 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import { runExtensionHook } from '../../src/evaluatorHelpers';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, mockApiProvider2, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator options and hooks', () => {
+  it('should use the options from the test if they exist', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { var1: 'value1', var2: 'value2' },
+          options: {
+            transform: 'output + " postprocessed"',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+  });
+
+  it('should apply prompt config to provider call', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test response',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [
+        {
+          raw: 'You are a helpful math tutor. Solve {{problem}}',
+          label: 'Math problem',
+          config: {
+            response_format: {
+              type: 'json_schema',
+              json_schema: {
+                name: 'math_response',
+                strict: true,
+                schema: {
+                  type: 'object',
+                  properties: {
+                    final_answer: { type: 'string' },
+                  },
+                  required: ['final_answer'],
+                  additionalProperties: false,
+                },
+              },
+            },
+          },
+        },
+      ],
+      tests: [{ vars: { problem: '8x + 31 = 2' } }],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(mockApiProvider.callApi).toHaveBeenCalledWith(
+      'You are a helpful math tutor. Solve 8x + 31 = 2',
+      expect.objectContaining({
+        prompt: expect.objectContaining({
+          config: {
+            response_format: {
+              type: 'json_schema',
+              json_schema: {
+                name: 'math_response',
+                schema: {
+                  type: 'object',
+                  properties: { final_answer: { type: 'string' } },
+                  required: ['final_answer'],
+                  additionalProperties: false,
+                },
+                strict: true,
+              },
+            },
+          },
+        }),
+      }),
+      undefined,
+    );
+  });
+
+  it('should apply dynamic prompt function config to provider call', async () => {
+    const mockDynamicConfigProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test response',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockDynamicConfigProvider],
+      prompts: [
+        {
+          raw: 'ignored by prompt function',
+          label: 'Dynamic prompt with config',
+          config: {
+            temperature: 0.1,
+            top_p: 0.9,
+          },
+          function: async () => ({
+            prompt: 'Solve this problem: {{problem}}',
+            config: {
+              temperature: 0.3,
+              response_format: { type: 'json_object' },
+            },
+          }),
+        },
+      ],
+      tests: [
+        {
+          vars: { problem: '8x + 31 = 2' },
+          options: { temperature: 0.7 },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(mockDynamicConfigProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(mockDynamicConfigProvider.callApi).toHaveBeenCalledWith(
+      'Solve this problem: 8x + 31 = 2',
+      expect.objectContaining({
+        prompt: expect.objectContaining({
+          config: {
+            temperature: 0.7,
+            top_p: 0.9,
+            response_format: { type: 'json_object' },
+          },
+        }),
+      }),
+      undefined,
+    );
+  });
+
+  it('should call runExtensionHook with correct parameters at appropriate times', async () => {
+    const mockExtension = 'file:./path/to/extension.js:extensionFunction';
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ var1 }}')],
+      tests: [
+        {
+          vars: { var1: 'value1' },
+          assert: [{ type: 'equals', value: 'Test output' }],
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockClear();
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    // Check if runExtensionHook was called 4 times (beforeAll, beforeEach, afterEach, afterAll)
+    expect(mockedRunExtensionHook).toHaveBeenCalledTimes(4);
+    // Check beforeAll call
+    expect(mockedRunExtensionHook).toHaveBeenNthCalledWith(
+      1,
+      [mockExtension],
+      'beforeAll',
+      expect.objectContaining({ suite: testSuite }),
+    );
+
+    // Check beforeEach call
+    expect(mockedRunExtensionHook).toHaveBeenNthCalledWith(
+      2,
+      [mockExtension],
+      'beforeEach',
+      expect.objectContaining({
+        test: expect.objectContaining({
+          vars: { var1: 'value1' },
+          assert: [{ type: 'equals', value: 'Test output' }],
+        }),
+      }),
+    );
+
+    // Check afterEach call
+    expect(mockedRunExtensionHook).toHaveBeenNthCalledWith(
+      3,
+      [mockExtension],
+      'afterEach',
+      expect.objectContaining({
+        test: expect.objectContaining({
+          vars: { var1: 'value1' },
+          assert: [{ type: 'equals', value: 'Test output' }],
+        }),
+        result: expect.objectContaining({
+          success: true,
+          score: 1,
+          response: expect.objectContaining({
+            output: 'Test output',
+          }),
+        }),
+      }),
+    );
+
+    // Check afterAll call
+    expect(mockedRunExtensionHook).toHaveBeenNthCalledWith(
+      4,
+      [mockExtension],
+      'afterAll',
+      expect.objectContaining({
+        prompts: expect.arrayContaining([
+          expect.objectContaining({
+            raw: 'Test prompt {{ var1 }}',
+            metrics: expect.objectContaining({
+              assertPassCount: 1,
+              assertFailCount: 0,
+            }),
+          }),
+          expect.objectContaining({
+            raw: 'Test prompt {{ var1 }}',
+            metrics: expect.objectContaining({
+              assertPassCount: 1,
+              assertFailCount: 0,
+            }),
+          }),
+        ]),
+        results: expect.any(Array),
+        suite: testSuite,
+      }),
+    );
+  });
+
+  it('should handle multiple providers', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider, mockApiProvider2],
+      prompts: [toPrompt('Test prompt')],
+      tests: [],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(summary.stats.failures).toBe(0);
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(mockApiProvider2.callApi).toHaveBeenCalledTimes(1);
+  });
+});

test/evaluator/repeatCache.test.ts

@@ -0,0 +1,452 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { clearCache, getCache } from '../../src/cache';
+import cliState from '../../src/cliState';
+import { evaluate, runEval } from '../../src/evaluator';
+import { runExtensionHook } from '../../src/evaluatorHelpers';
+import Eval from '../../src/models/eval';
+import {
+  type ApiProvider,
+  type Prompt,
+  type ProviderResponse,
+  ResultFailureReason,
+  type TestSuite,
+} from '../../src/types/index';
+import { createEmptyTokenUsage } from '../../src/util/tokenUsageUtils';
+import { toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator repeat cache isolation', () => {
+  it('preserves provider cache settings for repeat iterations', async () => {
+    const contexts: Array<Record<string, any> | undefined> = [];
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi
+        .fn()
+        .mockImplementation(async (_prompt: string, context?: Record<string, any>) => {
+          contexts.push(context);
+          return {
+            output: 'result',
+            tokenUsage: createEmptyTokenUsage(),
+          };
+        }),
+    };
+
+    const baseOptions = {
+      provider,
+      prompt: { raw: 'Test prompt', label: 'test-label' } as Prompt,
+      delay: 0,
+      nunjucksFilters: undefined,
+      evaluateOptions: {},
+      testIdx: 0,
+      promptIdx: 0,
+      conversations: {},
+      registers: {},
+      isRedteam: false,
+    };
+
+    await runEval({
+      ...baseOptions,
+      test: { assert: [] },
+      repeatIndex: 0,
+    });
+
+    expect(contexts).toHaveLength(1);
+    expect(contexts[0]).toBeDefined();
+    expect(contexts[0]!.bustCache).toBeFalsy();
+
+    contexts.length = 0;
+
+    await runEval({
+      ...baseOptions,
+      test: { assert: [] },
+      repeatIndex: 1,
+    });
+
+    expect(contexts).toHaveLength(1);
+    expect(contexts[0]).toBeDefined();
+    expect(contexts[0]!.bustCache).toBeFalsy();
+  });
+
+  it('isolates per-test repeat provider cache entries from the default namespace', async () => {
+    await clearCache();
+
+    const baselineResponse: ProviderResponse = {
+      output: 'baseline-result',
+      tokenUsage: createEmptyTokenUsage(),
+    };
+    await getCache().set('manual-provider-key', baselineResponse);
+
+    let cacheMissCount = 0;
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi
+        .fn()
+        .mockImplementation(async (_prompt: string, context?: Record<string, any>) => {
+          const cache = await context?.getCache();
+          const cachedResponse = await cache?.get('manual-provider-key');
+          if (cachedResponse) {
+            return {
+              ...(cachedResponse as ProviderResponse),
+              cached: true,
+            };
+          }
+
+          cacheMissCount += 1;
+          const response = {
+            cached: false,
+            output: `result-repeat-${context?.repeatIndex}-miss-${cacheMissCount}`,
+            tokenUsage: createEmptyTokenUsage(),
+          };
+          await cache?.set('manual-provider-key', response);
+          return response;
+        }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{ options: { repeat: 2 } }],
+    };
+
+    const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, firstEval, { maxConcurrency: 1 });
+    const firstSummary = await firstEval.toEvaluateSummary();
+
+    expect(cacheMissCount).toBe(2);
+    expect(firstSummary.results.map((result) => result.response?.output)).toEqual([
+      'result-repeat-0-miss-1',
+      'result-repeat-1-miss-2',
+    ]);
+    expect(firstSummary.results.map((result) => result.response?.cached)).toEqual([false, false]);
+
+    const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, secondEval, { maxConcurrency: 1 });
+    const secondSummary = await secondEval.toEvaluateSummary();
+
+    expect(cacheMissCount).toBe(2);
+    expect(secondSummary.results.map((result) => result.response?.output)).toEqual([
+      'result-repeat-0-miss-1',
+      'result-repeat-1-miss-2',
+    ]);
+    expect(secondSummary.results.map((result) => result.response?.cached)).toEqual([true, true]);
+    expect(await getCache().get('manual-provider-key')).toEqual(baselineResponse);
+  });
+
+  it('isolates beforeEach extension cache entries by repeat index', async () => {
+    await clearCache();
+
+    let extensionCacheMissCount = 0;
+    vi.mocked(runExtensionHook).mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName !== 'beforeEach' || !('test' in context)) {
+        return context;
+      }
+
+      const hookContext = context as typeof context & {
+        test: {
+          vars?: Record<string, unknown>;
+        };
+      };
+
+      const cache = getCache();
+      const cachedHookValue = await cache.get<string>('extension-hook-key');
+      if (cachedHookValue) {
+        return {
+          ...hookContext,
+          test: {
+            ...hookContext.test,
+            vars: {
+              ...hookContext.test.vars,
+              hookValue: cachedHookValue,
+            },
+          },
+        };
+      }
+
+      extensionCacheMissCount += 1;
+      const hookValue = `hook-repeat-${extensionCacheMissCount}`;
+      await cache.set('extension-hook-key', hookValue);
+
+      return {
+        ...hookContext,
+        test: {
+          ...hookContext.test,
+          vars: {
+            ...hookContext.test.vars,
+            hookValue,
+          },
+        },
+      };
+    });
+
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi
+        .fn()
+        .mockImplementation(async (_prompt: string, context?: Record<string, any>) => ({
+          cached: false,
+          output: context?.vars?.hookValue,
+          tokenUsage: createEmptyTokenUsage(),
+        })),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{ vars: {} }],
+      extensions: ['file://hook.js'],
+    };
+
+    const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, firstEval, { maxConcurrency: 1, repeat: 2 });
+    const firstSummary = await firstEval.toEvaluateSummary();
+
+    expect(extensionCacheMissCount).toBe(2);
+    expect(firstSummary.results.map((result) => result.response?.output)).toEqual([
+      'hook-repeat-1',
+      'hook-repeat-2',
+    ]);
+
+    const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, secondEval, { maxConcurrency: 1, repeat: 2 });
+    const secondSummary = await secondEval.toEvaluateSummary();
+
+    expect(extensionCacheMissCount).toBe(2);
+    expect(secondSummary.results.map((result) => result.response?.output)).toEqual([
+      'hook-repeat-1',
+      'hook-repeat-2',
+    ]);
+  });
+
+  it('isolates afterEach extension cache entries by repeat index', async () => {
+    await clearCache();
+
+    let extensionCacheMissCount = 0;
+    const afterEachHookValues: string[] = [];
+    vi.mocked(runExtensionHook).mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName !== 'afterEach' || !('result' in context)) {
+        return context;
+      }
+
+      const cache = getCache();
+      let hookValue = await cache.get<string>('after-each-extension-key');
+      if (!hookValue) {
+        extensionCacheMissCount += 1;
+        hookValue = `after-hook-repeat-${extensionCacheMissCount}`;
+        await cache.set('after-each-extension-key', hookValue);
+      }
+
+      afterEachHookValues.push(hookValue);
+      return context;
+    });
+
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        cached: false,
+        output: 'provider-output',
+        tokenUsage: createEmptyTokenUsage(),
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+      extensions: ['file://hook.js'],
+    };
+
+    const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, firstEval, { maxConcurrency: 1, repeat: 2 });
+
+    expect(extensionCacheMissCount).toBe(2);
+    expect(afterEachHookValues).toEqual(['after-hook-repeat-1', 'after-hook-repeat-2']);
+
+    afterEachHookValues.length = 0;
+
+    const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, secondEval, { maxConcurrency: 1, repeat: 2 });
+
+    expect(extensionCacheMissCount).toBe(2);
+    expect(afterEachHookValues).toEqual(['after-hook-repeat-1', 'after-hook-repeat-2']);
+  });
+
+  it('isolates select-best comparison cache entries by repeat index', async () => {
+    await clearCache();
+
+    const matchers = await import('../../src/matchers/comparison');
+    let comparisonCacheMissCount = 0;
+    const comparisonRepeatIndexes: Array<number | undefined> = [];
+    const matchesSelectBestSpy = vi
+      .spyOn(matchers, 'matchesSelectBest')
+      .mockImplementation(async (_criteria, outputs, _grading, _vars, context) => {
+        comparisonRepeatIndexes.push(context?.repeatIndex);
+        const cache = context?.getCache?.() ?? getCache();
+        const cachedResult = (await cache.get('select-best-comparison-key')) as string | undefined;
+        if (!cachedResult) {
+          comparisonCacheMissCount += 1;
+          await cache.set('select-best-comparison-key', `repeat-${context?.repeatIndex}`);
+        }
+
+        return outputs.map((_output, index) => ({
+          pass: index === 0,
+          score: index === 0 ? 1 : 0,
+          reason: index === 0 ? 'Selected as best' : 'Not selected',
+        }));
+      });
+
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi
+        .fn()
+        .mockImplementation(async (prompt: string, context?: Record<string, any>) => ({
+          cached: false,
+          output: `${prompt}-repeat-${context?.repeatIndex}`,
+          tokenUsage: createEmptyTokenUsage(),
+        })),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Prompt A'), toPrompt('Prompt B')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'select-best',
+              value: 'choose the best one',
+            },
+          ],
+        },
+      ],
+    };
+
+    try {
+      const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+      await evaluate(testSuite, firstEval, { maxConcurrency: 1, repeat: 2 });
+
+      expect(comparisonCacheMissCount).toBe(2);
+      expect(matchesSelectBestSpy).toHaveBeenCalledTimes(2);
+      expect(comparisonRepeatIndexes).toEqual([0, 1]);
+
+      const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+      await evaluate(testSuite, secondEval, { maxConcurrency: 1, repeat: 2 });
+
+      expect(comparisonCacheMissCount).toBe(2);
+      expect(matchesSelectBestSpy).toHaveBeenCalledTimes(4);
+      expect(comparisonRepeatIndexes).toEqual([0, 1, 0, 1]);
+    } finally {
+      matchesSelectBestSpy.mockRestore();
+    }
+  });
+
+  it('isolates resumed select-best comparison cache entries by original repeat index', async () => {
+    await clearCache();
+
+    const originalResume = cliState.resume;
+    const { default: EvalResultModel } = await import('../../src/models/evalResult');
+    const getCompletedIndexPairsSpy = vi
+      .spyOn(EvalResultModel, 'getCompletedIndexPairs')
+      .mockResolvedValue(new Set(['0:0', '0:1', '1:0', '1:1']));
+
+    const matchers = await import('../../src/matchers/comparison');
+    let comparisonCacheMissCount = 0;
+    const comparisonRepeatIndexes: Array<number | undefined> = [];
+    const matchesSelectBestSpy = vi
+      .spyOn(matchers, 'matchesSelectBest')
+      .mockImplementation(async (_criteria, outputs, _grading, _vars, context) => {
+        comparisonRepeatIndexes.push(context?.repeatIndex);
+        const cache = context?.getCache?.() ?? getCache();
+        const cachedResult = (await cache.get('resume-select-best-key')) as string | undefined;
+        if (!cachedResult) {
+          comparisonCacheMissCount += 1;
+          await cache.set('resume-select-best-key', `repeat-${context?.repeatIndex}`);
+        }
+
+        return outputs.map((_output, index) => ({
+          pass: index === 0,
+          score: index === 0 ? 1 : 0,
+          reason: index === 0 ? 'Selected as best' : 'Not selected',
+        }));
+      });
+
+    const provider: ApiProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        cached: false,
+        output: 'should be skipped by resume',
+        tokenUsage: createEmptyTokenUsage(),
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider],
+      prompts: [toPrompt('Prompt A'), toPrompt('Prompt B')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'select-best',
+              value: 'choose the best one',
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    const fetchResultsByTestIdxSpy = vi
+      .spyOn(evalRecord, 'fetchResultsByTestIdx')
+      .mockImplementation(
+        async (testIdx: number) =>
+          testSuite.prompts.map((prompt, promptIdx) => ({
+            failureReason: ResultFailureReason.NONE,
+            gradingResult: {
+              componentResults: [],
+              pass: true,
+              reason: 'Existing result',
+              score: 1,
+            },
+            prompt,
+            promptIdx,
+            provider: { id: provider.id() },
+            response: {
+              output: `${prompt.raw}-test-${testIdx}`,
+              tokenUsage: createEmptyTokenUsage(),
+            },
+            save: vi.fn().mockResolvedValue(undefined),
+            score: 1,
+            success: true,
+            testCase: {
+              ...testSuite.tests![0],
+              vars: {},
+            },
+            testIdx,
+          })) as any,
+      );
+
+    evalRecord.persisted = true;
+    cliState.resume = true;
+
+    try {
+      await evaluate(testSuite, evalRecord, { maxConcurrency: 1, repeat: 2 });
+
+      expect(getCompletedIndexPairsSpy).toHaveBeenCalledWith(evalRecord.id, {
+        excludeErrors: cliState.retryMode,
+      });
+      expect(fetchResultsByTestIdxSpy).toHaveBeenCalledTimes(2);
+      expect(provider.callApi).not.toHaveBeenCalled();
+      expect(comparisonCacheMissCount).toBe(2);
+      expect(matchesSelectBestSpy).toHaveBeenCalledTimes(2);
+      expect(comparisonRepeatIndexes).toEqual([0, 1]);
+    } finally {
+      cliState.resume = originalResume;
+      getCompletedIndexPairsSpy.mockRestore();
+      matchesSelectBestSpy.mockRestore();
+      fetchResultsByTestIdxSpy.mockRestore();
+    }
+  });
+});

test/evaluator/routing.test.ts

@@ -0,0 +1,499 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator prompt and provider routing', () => {
+  it('evaluate with providerPromptMap', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt 1'), toPrompt('Test prompt 2')],
+      providerPromptMap: {
+        'test-provider': ['Test prompt 1'],
+      },
+      tests: [
+        {
+          vars: { var1: 'value1', var2: 'value2' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 0,
+        prompt: 0,
+        completion: 0,
+        cached: 0,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+    expect(summary.results[0].prompt.raw).toBe('Test prompt 1');
+    expect(summary.results[0].prompt.label).toBe('Test prompt 1');
+    expect(summary.results[0].response?.output).toBe('Test output');
+  });
+
+  it('evaluate with allowed prompts filtering', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [
+        { raw: 'Test prompt 1', label: 'prompt1' },
+        { raw: 'Test prompt 2', label: 'prompt2' },
+        { raw: 'Test prompt 3', label: 'group1:prompt3' },
+      ],
+      providerPromptMap: {
+        'test-provider': ['prompt1', 'group1'],
+      },
+      tests: [
+        {
+          vars: { var1: 'value1', var2: 'value2' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary).toMatchObject({
+      stats: {
+        successes: 2,
+        failures: 0,
+      },
+      results: [{ prompt: { label: 'prompt1' } }, { prompt: { label: 'group1:prompt3' } }],
+    });
+  });
+
+  it('evaluate with labeled and unlabeled providers and providerPromptMap', async () => {
+    const mockLabeledProvider: ApiProvider = {
+      id: () => 'labeled-provider-id',
+      label: 'Labeled Provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Labeled Provider Output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const mockUnlabeledProvider: ApiProvider = {
+      id: () => 'unlabeled-provider-id',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Unlabeled Provider Output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockLabeledProvider, mockUnlabeledProvider],
+      prompts: [
+        {
+          raw: 'Prompt 1',
+          label: 'prompt1',
+        },
+        {
+          raw: 'Prompt 2',
+          label: 'prompt2',
+        },
+      ],
+      providerPromptMap: {
+        'Labeled Provider': ['prompt1'],
+        'unlabeled-provider-id': ['prompt2'],
+      },
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary).toMatchObject({
+      stats: expect.objectContaining({
+        successes: 2,
+        failures: 0,
+      }),
+      results: [
+        expect.objectContaining({
+          provider: expect.objectContaining({
+            id: 'labeled-provider-id',
+            label: 'Labeled Provider',
+          }),
+          response: expect.objectContaining({
+            output: 'Labeled Provider Output',
+          }),
+        }),
+        expect.objectContaining({
+          provider: expect.objectContaining({
+            id: 'unlabeled-provider-id',
+            label: undefined,
+          }),
+          response: expect.objectContaining({
+            output: 'Unlabeled Provider Output',
+          }),
+        }),
+      ],
+    });
+    expect(evalRecord.prompts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          provider: 'Labeled Provider',
+        }),
+        expect.objectContaining({
+          provider: 'unlabeled-provider-id',
+        }),
+      ]),
+    );
+
+    expect(mockLabeledProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(mockUnlabeledProvider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('evaluate with test-level providers filter', async () => {
+    const mockProvider1: ApiProvider = {
+      id: () => 'provider-1',
+      label: 'fast-model',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Fast Output',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const mockProvider2: ApiProvider = {
+      id: () => 'provider-2',
+      label: 'smart-model',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Smart Output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockProvider1, mockProvider2],
+      prompts: [{ raw: 'Test prompt', label: 'prompt1' }],
+      tests: [
+        {
+          description: 'fast test',
+          vars: { input: 'simple' },
+          providers: ['fast-model'], // Only run on fast-model
+        },
+        {
+          description: 'smart test',
+          vars: { input: 'complex' },
+          providers: ['smart-model'], // Only run on smart-model
+        },
+        {
+          description: 'all providers test',
+          vars: { input: 'general' },
+          // No providers filter - runs on both
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // 1 test on fast-model + 1 test on smart-model + 2 tests (1 on each provider) = 4 total
+    expect(summary.stats.successes).toBe(4);
+    expect(mockProvider1.callApi).toHaveBeenCalledTimes(2); // fast test + all providers test
+    expect(mockProvider2.callApi).toHaveBeenCalledTimes(2); // smart test + all providers test
+  });
+
+  it('evaluate with test-level providers filter using wildcard', async () => {
+    const openaiProvider: ApiProvider = {
+      id: () => 'openai:gpt-4',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'OpenAI Output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const anthropicProvider: ApiProvider = {
+      id: () => 'anthropic:claude-3',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Anthropic Output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [openaiProvider, anthropicProvider],
+      prompts: [{ raw: 'Test prompt', label: 'prompt1' }],
+      tests: [
+        {
+          vars: { input: 'test' },
+          providers: ['openai:*'], // Wildcard - only run on openai providers
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(1);
+    expect(openaiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(anthropicProvider.callApi).toHaveBeenCalledTimes(0);
+  });
+
+  it('evaluate inherits providers filter from defaultTest', async () => {
+    const provider1: ApiProvider = {
+      id: () => 'provider-1',
+      label: 'default-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output 1',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const provider2: ApiProvider = {
+      id: () => 'provider-2',
+      label: 'other-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output 2',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider1, provider2],
+      prompts: [{ raw: 'Test prompt', label: 'prompt1' }],
+      defaultTest: {
+        providers: ['default-provider'], // Default to only default-provider
+      },
+      tests: [
+        {
+          vars: { input: 'test1' },
+          // Inherits providers filter from defaultTest
+        },
+        {
+          vars: { input: 'test2' },
+          providers: ['other-provider'], // Override defaultTest
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(provider1.callApi).toHaveBeenCalledTimes(1); // test1 only
+    expect(provider2.callApi).toHaveBeenCalledTimes(1); // test2 only
+  });
+
+  it('evaluate with empty providers array blocks all providers', async () => {
+    const mockProvider: ApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockProvider],
+      prompts: [{ raw: 'Test prompt', label: 'prompt1' }],
+      tests: [
+        {
+          vars: { input: 'test' },
+          providers: [], // Empty array = block all
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(0);
+    expect(mockProvider.callApi).toHaveBeenCalledTimes(0);
+  });
+
+  it('evaluate with providers filter and providerPromptMap combined', async () => {
+    const provider1: ApiProvider = {
+      id: () => 'provider-1',
+      label: 'provider-one',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output 1',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const provider2: ApiProvider = {
+      id: () => 'provider-2',
+      label: 'provider-two',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output 2',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider1, provider2],
+      prompts: [
+        { raw: 'Prompt A', label: 'prompt-a' },
+        { raw: 'Prompt B', label: 'prompt-b' },
+      ],
+      providerPromptMap: {
+        'provider-one': ['prompt-a'], // provider-one only runs prompt-a
+        'provider-two': ['prompt-b'], // provider-two only runs prompt-b
+      },
+      tests: [
+        {
+          vars: { input: 'test' },
+          providers: ['provider-one'], // Only run on provider-one
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // providers filter limits to provider-one
+    // providerPromptMap limits provider-one to prompt-a
+    // Result: 1 test case (provider-one + prompt-a)
+    expect(summary.stats.successes).toBe(1);
+    expect(provider1.callApi).toHaveBeenCalledTimes(1);
+    expect(provider2.callApi).toHaveBeenCalledTimes(0);
+  });
+
+  it('promptIdx aligns with prompts array when test-level providers filter skips providers', async () => {
+    // This test verifies that promptIdx correctly maps to the prompts array
+    // even when test-level provider filtering causes some providers to be skipped.
+    // Before the fix, promptIdx was a sequential counter that could misalign with
+    // the prompts array when filters caused gaps.
+    const provider1: ApiProvider = {
+      id: () => 'provider-1',
+      label: 'model-a',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output from model-a',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const provider2: ApiProvider = {
+      id: () => 'provider-2',
+      label: 'model-b',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output from model-b',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider1, provider2],
+      prompts: [toPrompt('Prompt X'), toPrompt('Prompt Y')],
+      tests: [
+        {
+          vars: { input: 'test' },
+          providers: ['model-b'], // Only run on provider2, skip provider1
+        },
+      ],
+    };
+
+    // prompts array should be: [model-a+PromptX(0), model-a+PromptY(1), model-b+PromptX(2), model-b+PromptY(3)]
+    // With provider filter, only model-b runs, so promptIdx should be 2 and 3 (not 0 and 1)
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(provider1.callApi).toHaveBeenCalledTimes(0);
+    expect(provider2.callApi).toHaveBeenCalledTimes(2);
+
+    // Verify promptIdx values map to the correct provider in the prompts array
+    const results = summary.results.sort((a, b) => a.promptIdx - b.promptIdx);
+    expect(results).toHaveLength(2);
+    // Both results should have promptIdx >= 2 (mapping to provider2's entries)
+    for (const result of results) {
+      expect(result.promptIdx).toBeGreaterThanOrEqual(2);
+      expect(result.provider.id).toBe('provider-2');
+    }
+  });
+
+  it('promptIdx aligns with prompts array when test-level prompts filter skips prompts', async () => {
+    const provider1: ApiProvider = {
+      id: () => 'provider-1',
+      label: 'model-a',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output from model-a',
+        tokenUsage: { total: 5, prompt: 2, completion: 3, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const provider2: ApiProvider = {
+      id: () => 'provider-2',
+      label: 'model-b',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Output from model-b',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [provider1, provider2],
+      prompts: [
+        { raw: 'Prompt X', label: 'promptX' },
+        { raw: 'Prompt Y', label: 'promptY' },
+      ],
+      tests: [
+        {
+          vars: { input: 'test' },
+          prompts: ['promptY'], // Only run with Prompt Y, skip Prompt X
+        },
+      ],
+    };
+
+    // prompts array: [model-a+promptX(0), model-a+promptY(1), model-b+promptX(2), model-b+promptY(3)]
+    // With prompt filter, only promptY runs, so promptIdx should be 1 and 3 (not 0 and 1)
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(2);
+    expect(provider1.callApi).toHaveBeenCalledTimes(1);
+    expect(provider2.callApi).toHaveBeenCalledTimes(1);
+
+    // Verify promptIdx values map to the correct prompt entries (odd indices for promptY)
+    const results = summary.results.sort((a, b) => a.promptIdx - b.promptIdx);
+    expect(results).toHaveLength(2);
+    // promptIdx should be 1 (model-a+promptY) and 3 (model-b+promptY)
+    expect(results[0].promptIdx).toBe(1);
+    expect(results[1].promptIdx).toBe(3);
+  });
+});

test/evaluator/runtime.test.ts

@@ -0,0 +1,308 @@
+import './setup';
+
+import { randomUUID } from 'node:crypto';
+
+import { expect, it, vi } from 'vitest';
+import cliState from '../../src/cliState';
+import { evaluate } from '../../src/evaluator';
+import {
+  type InMemoryEvaluation,
+  InMemoryEvaluationStore,
+} from '../../src/evaluator/inMemoryStore';
+import logger from '../../src/logger';
+import Eval from '../../src/models/eval';
+import { EvalEvaluationStore } from '../../src/node/evaluationStore';
+import { ResultFailureReason, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+import type { EvaluationStore, EvaluatorRuntime } from '../../src/evaluator/runtime';
+import type EvalResult from '../../src/models/evalResult';
+import type { ApiProvider, EvaluateResult } from '../../src/types/index';
+
+function createResultWriter() {
+  return {
+    write: vi.fn().mockResolvedValue(undefined),
+    close: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+function createRuntime(resultWriters = [createResultWriter()]): EvaluatorRuntime<Eval, EvalResult> {
+  return {
+    createEvaluationStore: vi.fn((evaluation) => new EvalEvaluationStore(evaluation)),
+    createResultWriters: vi.fn().mockReturnValue(resultWriters),
+  };
+}
+
+function createEvalRecord(): Eval {
+  return new Eval({ outputPath: 'results.jsonl' }, { id: randomUUID(), persisted: false });
+}
+
+function createInMemoryRuntime(
+  store: EvaluationStore<InMemoryEvaluation, EvaluateResult>,
+): EvaluatorRuntime<InMemoryEvaluation, EvaluateResult> {
+  return {
+    createEvaluationStore: vi.fn().mockReturnValue(store),
+    createResultWriters: vi.fn().mockReturnValue([]),
+  };
+}
+
+function createInMemoryEvaluation(overrides: Partial<InMemoryEvaluation> = {}): InMemoryEvaluation {
+  return {
+    id: 'in-memory-eval',
+    config: {},
+    persisted: false,
+    prompts: [],
+    results: [],
+    vars: [],
+    resultPersistenceFailed: false,
+    finalResults: [],
+    failedResults: [],
+    ...overrides,
+  };
+}
+
+describeEvaluator('evaluator runtime ports', () => {
+  it('evaluates with an in-memory store and preserves evaluation identity', async () => {
+    const evaluation = createInMemoryEvaluation();
+    const store = new InMemoryEvaluationStore(evaluation);
+    const runtime = createInMemoryRuntime(store);
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+
+    await expect(evaluate(testSuite, evaluation, {}, runtime)).resolves.toBe(evaluation);
+
+    expect(evaluation.results).toHaveLength(1);
+    expect(evaluation.results[0]).toMatchObject({
+      success: true,
+      testIdx: 0,
+      promptIdx: 0,
+    });
+    expect(evaluation.prompts).toHaveLength(1);
+  });
+
+  it('uses the store resume lookup without importing a concrete result model', async () => {
+    const evaluation = createInMemoryEvaluation({
+      persisted: true,
+      results: [
+        {
+          ...({} as EvaluateResult),
+          failureReason: ResultFailureReason.NONE,
+          promptIdx: 0,
+          testIdx: 0,
+        },
+      ],
+    });
+    const store = new InMemoryEvaluationStore(evaluation);
+    const readCompletedIndexPairs = vi.spyOn(store, 'readCompletedIndexPairs');
+    const runtime = createInMemoryRuntime(store);
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+    cliState.resume = true;
+    cliState.retryMode = false;
+
+    await evaluate(testSuite, evaluation, {}, runtime);
+
+    expect(readCompletedIndexPairs).toHaveBeenCalledWith({ excludeErrors: false });
+    expect(evaluation.results).toHaveLength(1);
+  });
+
+  it('persists comparison updates through an explicit in-memory runtime', async () => {
+    const evaluation = createInMemoryEvaluation({ persisted: true });
+    const store = new InMemoryEvaluationStore(evaluation);
+    const runtime = createInMemoryRuntime(store);
+    const maxScoreProvider: ApiProvider = {
+      id: () => 'max-score-provider',
+      callApi: vi.fn().mockResolvedValue({ output: 'hello world' }),
+    };
+    const testSuite: TestSuite = {
+      providers: [maxScoreProvider],
+      prompts: [toPrompt('Prompt A'), toPrompt('Prompt B')],
+      tests: [
+        {
+          assert: [{ type: 'contains', value: 'hello' }, { type: 'max-score' }],
+        },
+      ],
+    };
+
+    await evaluate(testSuite, evaluation, {}, runtime);
+
+    const results = [...evaluation.results].sort((left, right) => left.promptIdx - right.promptIdx);
+    expect(results).toHaveLength(2);
+    expect(results[0].success).toBe(true);
+    expect(results[1]).toMatchObject({
+      success: false,
+      failureReason: ResultFailureReason.ASSERT,
+    });
+  });
+
+  it('requires an explicit runtime for non-default evaluation records', () => {
+    if (false) {
+      // @ts-expect-error Custom evaluation records must provide their own runtime.
+      void evaluate({} as TestSuite, createInMemoryEvaluation(), {});
+    }
+  });
+
+  it('delegates result side effects and closes writers during cleanup', async () => {
+    const resultWriter = createResultWriter();
+    const runtime = createRuntime([resultWriter]);
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+    const evalRecord = createEvalRecord();
+    const appendResult = vi.spyOn(evalRecord, 'addResult');
+
+    await evaluate(testSuite, evalRecord, {}, runtime);
+
+    expect(runtime.createEvaluationStore).toHaveBeenCalledWith(evalRecord);
+    expect(runtime.createResultWriters).toHaveBeenCalledWith('results.jsonl', { append: false });
+    expect(appendResult).toHaveBeenCalledOnce();
+    expect(resultWriter.write).toHaveBeenCalledOnce();
+    expect(appendResult.mock.invocationCallOrder[0]).toBeLessThan(
+      resultWriter.write.mock.invocationCallOrder[0],
+    );
+    expect(resultWriter.close).toHaveBeenCalledOnce();
+  });
+
+  it('passes resume append semantics to result writers', async () => {
+    const runtime = createRuntime([]);
+    const testSuite: TestSuite = {
+      providers: [],
+      prompts: [],
+      tests: [],
+    };
+    cliState.resume = true;
+
+    await evaluate(testSuite, createEvalRecord(), {}, runtime);
+
+    expect(runtime.createResultWriters).toHaveBeenCalledWith('results.jsonl', { append: true });
+  });
+
+  it('continues streaming output when result persistence fails', async () => {
+    const resultWriter = createResultWriter();
+    const runtime = createRuntime([resultWriter]);
+    const evalRecord = createEvalRecord();
+    vi.spyOn(evalRecord, 'addResult').mockRejectedValue(new Error('database unavailable'));
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+
+    await expect(evaluate(testSuite, evalRecord, {}, runtime)).resolves.toBeDefined();
+
+    expect(resultWriter.write).toHaveBeenCalledOnce();
+    expect(resultWriter.close).toHaveBeenCalledOnce();
+  });
+
+  it('rejects output failures after closing writers', async () => {
+    const resultWriter = createResultWriter();
+    resultWriter.write.mockRejectedValue(new Error('output unavailable'));
+    const runtime = createRuntime([resultWriter]);
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+
+    await expect(evaluate(testSuite, createEvalRecord(), {}, runtime)).rejects.toThrow(
+      'output unavailable',
+    );
+
+    expect(resultWriter.close).toHaveBeenCalledOnce();
+  });
+
+  it('attempts every writer and recovers a close failure when results persisted', async () => {
+    const failingWriter = createResultWriter();
+    failingWriter.close.mockRejectedValue(new Error('close unavailable'));
+    const healthyWriter = createResultWriter();
+    const runtime = createRuntime([failingWriter, healthyWriter]);
+    const warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => logger);
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+
+    try {
+      // Results persisted to the DB, so the post-run rewrite regenerates the JSONL from it;
+      // a close error is logged, not fatal to an otherwise-successful run.
+      await expect(evaluate(testSuite, createEvalRecord(), {}, runtime)).resolves.toBeDefined();
+      expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('close unavailable'));
+    } finally {
+      warnSpy.mockRestore();
+    }
+
+    expect(failingWriter.close).toHaveBeenCalledOnce();
+    expect(healthyWriter.close).toHaveBeenCalledOnce();
+  });
+
+  it('surfaces a close failure when result persistence also failed', async () => {
+    const failingWriter = createResultWriter();
+    failingWriter.close.mockRejectedValue(new Error('close unavailable'));
+    const runtime = createRuntime([failingWriter]);
+    const evalRecord = createEvalRecord();
+    // Persistence failed, so the streamed JSONL is the only copy of the results and a
+    // close error (possible truncation) must not be swallowed.
+    vi.spyOn(evalRecord, 'addResult').mockRejectedValue(new Error('database unavailable'));
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+
+    await expect(evaluate(testSuite, evalRecord, {}, runtime)).rejects.toThrow('close unavailable');
+
+    expect(failingWriter.close).toHaveBeenCalledOnce();
+  });
+
+  it('persists per-call timeout rows without streaming them', async () => {
+    vi.useFakeTimers();
+    const resultWriter = createResultWriter();
+    const runtime = createRuntime([resultWriter]);
+    const evalRecord = createEvalRecord();
+    const appendResult = vi.spyOn(evalRecord, 'addResult');
+    const slowProvider: ApiProvider = {
+      id: () => 'slow-provider',
+      callApi: vi.fn<ApiProvider['callApi']>((_prompt, _context, options) => {
+        return new Promise<never>((_resolve, reject) => {
+          const rejectAbort = () => {
+            const error = new Error('Operation aborted');
+            error.name = 'AbortError';
+            reject(error);
+          };
+          if (options?.abortSignal?.aborted) {
+            rejectAbort();
+            return;
+          }
+          options?.abortSignal?.addEventListener('abort', rejectAbort, { once: true });
+        });
+      }),
+    };
+    const testSuite: TestSuite = {
+      providers: [slowProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [{}],
+    };
+
+    try {
+      const evaluation = evaluate(testSuite, evalRecord, { timeoutMs: 10 }, runtime);
+      await vi.advanceTimersByTimeAsync(10);
+      await evaluation;
+
+      expect(appendResult).toHaveBeenCalledOnce();
+      expect(resultWriter.write).not.toHaveBeenCalled();
+      expect(resultWriter.close).toHaveBeenCalledOnce();
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+});

test/evaluator/scenarios.test.ts

@@ -0,0 +1,398 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import { toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator scenarios and conversations', () => {
+  it('evaluate with scenarios', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi
+        .fn()
+        .mockResolvedValueOnce({
+          output: 'Hola mundo',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        })
+        .mockResolvedValueOnce({
+          output: 'Bonjour le monde',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ language }}')],
+      scenarios: [
+        {
+          config: [
+            {
+              vars: {
+                language: 'Spanish',
+                expectedHelloWorld: 'Hola mundo',
+              },
+            },
+            {
+              vars: {
+                language: 'French',
+                expectedHelloWorld: 'Bonjour le monde',
+              },
+            },
+          ],
+          tests: [
+            {
+              assert: [
+                {
+                  type: 'equals',
+                  value: '{{expectedHelloWorld}}',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.stats.successes).toBe(2);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Hola mundo');
+    expect(summary.results[1].response?.output).toBe('Bonjour le monde');
+  });
+
+  it('applies repeat from scenario config options', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      scenarios: [
+        {
+          config: [{ options: { repeat: 3 } }],
+          tests: [{}],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+    await evaluate(testSuite, evalRecord, { repeat: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
+    expect(summary.results).toHaveLength(3);
+  });
+
+  it('lets scenario test options.repeat override scenario config options.repeat', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      scenarios: [
+        {
+          config: [{ options: { repeat: 5 } }],
+          tests: [{ options: { repeat: 2 } }],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+    await evaluate(testSuite, evalRecord, { repeat: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.results).toHaveLength(2);
+  });
+
+  it('evaluate with scenarios and multiple vars', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi
+        .fn()
+        .mockResolvedValueOnce({
+          output: 'Spanish Hola',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        })
+        .mockResolvedValueOnce({
+          output: 'Spanish Bonjour',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        })
+        .mockResolvedValueOnce({
+          output: 'French Hola',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        })
+        .mockResolvedValueOnce({
+          output: 'French Bonjour',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+        }),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ language }} {{ greeting }}')],
+      scenarios: [
+        {
+          config: [
+            {
+              vars: {
+                language: ['Spanish', 'French'],
+                greeting: ['Hola', 'Bonjour'],
+              },
+            },
+          ],
+          tests: [
+            {
+              assert: [
+                {
+                  type: 'equals',
+                  value: '{{language}} {{greeting}}',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+    expect(summary.stats.successes).toBe(4);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Spanish Hola');
+    expect(summary.results[1].response?.output).toBe('Spanish Bonjour');
+    expect(summary.results[2].response?.output).toBe('French Hola');
+    expect(summary.results[3].response?.output).toBe('French Bonjour');
+  });
+
+  it('evaluate with scenarios and defaultTest', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello, World',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      defaultTest: {
+        metadata: { defaultKey: 'defaultValue' },
+        assert: [
+          {
+            type: 'starts-with',
+            value: 'Hello',
+          },
+        ],
+      },
+      scenarios: [
+        {
+          config: [{ metadata: { configKey: 'configValue' } }],
+          tests: [{ metadata: { testKey: 'testValue' } }],
+        },
+        {
+          config: [
+            {
+              assert: [
+                {
+                  type: 'contains',
+                  value: ',',
+                },
+              ],
+            },
+          ],
+          tests: [
+            {
+              assert: [
+                {
+                  type: 'icontains',
+                  value: 'world',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary).toMatchObject({
+      stats: {
+        successes: 2,
+        failures: 0,
+      },
+      results: expect.arrayContaining([
+        expect.objectContaining({
+          gradingResult: expect.objectContaining({
+            componentResults: expect.arrayContaining([expect.anything()]),
+          }),
+        }),
+        expect.objectContaining({
+          gradingResult: expect.objectContaining({
+            componentResults: expect.arrayContaining([
+              expect.anything(),
+              expect.anything(),
+              expect.anything(),
+            ]),
+          }),
+        }),
+      ]),
+    });
+
+    expect(summary.results[0].testCase.metadata).toEqual({
+      defaultKey: 'defaultValue',
+      configKey: 'configValue',
+      testKey: 'testValue',
+      conversationId: '__scenario_0__', // Auto-generated for scenario isolation
+    });
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+  });
+
+  it('should maintain separate conversation histories between scenarios without explicit conversationId', async () => {
+    // This test verifies the fix for GitHub issue #384:
+    // Scenarios should have isolated _conversation state by default
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockImplementation((_prompt) => ({
+        output: 'Test output',
+      })),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [
+        {
+          raw: '{% for completion in _conversation %}Previous: {{ completion.input }} -> {{ completion.output }}\n{% endfor %}Current: {{ question }}',
+          label: 'Conversation test',
+        },
+      ],
+      scenarios: [
+        {
+          // First scenario - conversation about books
+          config: [{}],
+          tests: [
+            { vars: { question: 'Recommend a sci-fi book' } },
+            { vars: { question: 'Tell me more about it' } },
+          ],
+        },
+        {
+          // Second scenario - conversation about recipes
+          // Should NOT include history from first scenario
+          config: [{}],
+          tests: [
+            { vars: { question: 'Suggest a pasta recipe' } },
+            { vars: { question: 'How long does it take?' } },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+
+    // First scenario, first question - no history
+    const firstCall = mockApiProvider.callApi.mock.calls[0][0];
+    expect(firstCall).toContain('Current: Recommend a sci-fi book');
+    expect(firstCall).not.toContain('Previous:');
+
+    // First scenario, second question - should have first scenario's history
+    const secondCall = mockApiProvider.callApi.mock.calls[1][0];
+    expect(secondCall).toContain('Previous: ');
+    expect(secondCall).toContain('Recommend a sci-fi book');
+    expect(secondCall).toContain('Current: Tell me more about it');
+
+    // Second scenario, first question - should NOT have first scenario's history
+    // This is the key assertion that verifies the fix for issue #384
+    const thirdCall = mockApiProvider.callApi.mock.calls[2][0];
+    expect(thirdCall).toContain('Current: Suggest a pasta recipe');
+    expect(thirdCall).not.toContain('Previous:');
+    expect(thirdCall).not.toContain('sci-fi');
+    expect(thirdCall).not.toContain('Recommend');
+
+    // Second scenario, second question - should only have second scenario's history
+    const fourthCall = mockApiProvider.callApi.mock.calls[3][0];
+    expect(fourthCall).toContain('Previous: ');
+    expect(fourthCall).toContain('Suggest a pasta recipe');
+    expect(fourthCall).toContain('Current: How long does it take?');
+    expect(fourthCall).not.toContain('sci-fi');
+  });
+
+  it('should allow scenarios to share conversation history with explicit conversationId', async () => {
+    // This test verifies that users can still explicitly share conversations across scenarios
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockImplementation((_prompt) => ({
+        output: 'Test output',
+      })),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [
+        {
+          raw: '{% for completion in _conversation %}Previous: {{ completion.input }}\n{% endfor %}Current: {{ question }}',
+          label: 'Conversation test',
+        },
+      ],
+      scenarios: [
+        {
+          config: [{}],
+          tests: [
+            {
+              vars: { question: 'Question from scenario 1' },
+              metadata: { conversationId: 'shared-conversation' },
+            },
+          ],
+        },
+        {
+          config: [{}],
+          tests: [
+            {
+              vars: { question: 'Question from scenario 2' },
+              metadata: { conversationId: 'shared-conversation' },
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+
+    // First scenario - no history
+    const firstCall = mockApiProvider.callApi.mock.calls[0][0];
+    expect(firstCall).not.toContain('Previous:');
+
+    // Second scenario - SHOULD have first scenario's history because they share conversationId
+    const secondCall = mockApiProvider.callApi.mock.calls[1][0];
+    expect(secondCall).toContain('Previous: ');
+    expect(secondCall).toContain('Question from scenario 1');
+  });
+});

test/evaluator/select-best-minimal.integration.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it, vi } from 'vitest';
+import { runCompareAssertion } from '../../src/assertions/index';
+import { createMockProvider, createProviderResponse } from '../factories/provider';
+
+import type {
+  ApiProvider,
+  Assertion,
+  AtomicTestCase,
+  CallApiContextParams,
+} from '../../src/types/index';
+
+describe('select-best context propagation', () => {
+  it('should pass context with originalProvider to grading provider', async () => {
+    let capturedContext: CallApiContextParams | undefined;
+
+    // Create a grading provider that captures the context
+    const gradingProvider = createMockProvider({
+      id: 'test-grading-provider',
+      callApi: vi.fn<ApiProvider['callApi']>().mockImplementation(async (_prompt, context) => {
+        capturedContext = context;
+        return {
+          output: '0', // Select first option
+          tokenUsage: {},
+        };
+      }),
+    });
+
+    const originalProvider = createMockProvider({
+      id: 'original-provider',
+      response: createProviderResponse({ output: '', tokenUsage: {} }),
+    });
+
+    const test: AtomicTestCase = {
+      vars: { foo: 'bar' },
+      options: {
+        provider: gradingProvider,
+      },
+    };
+
+    const assertion: Assertion = {
+      type: 'select-best',
+      value: 'Pick the best output',
+    };
+
+    const outputs = ['Output 1', 'Output 2'];
+
+    const contextToPass: CallApiContextParams = {
+      originalProvider,
+      prompt: { raw: 'test prompt', label: 'test' },
+      vars: { foo: 'bar' },
+    };
+
+    // Call runCompareAssertion with context
+    await runCompareAssertion(test, assertion, outputs, contextToPass);
+
+    // Verify the grading provider was called
+    expect(gradingProvider.callApi).toHaveBeenCalled();
+
+    // Verify it received the context with originalProvider
+    expect(capturedContext).toBeDefined();
+    expect(capturedContext?.originalProvider).toBeDefined();
+    expect(capturedContext?.originalProvider?.id()).toBe('original-provider');
+  });
+});

test/evaluator/sessions.test.ts

@@ -0,0 +1,130 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import { runExtensionHook } from '../../src/evaluatorHelpers';
+import Eval from '../../src/models/eval';
+import { type TestSuite } from '../../src/types/index';
+import { toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator session metadata', () => {
+  it('should use sessionId from vars if not in response', async () => {
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        // No sessionId in response
+      }),
+    };
+
+    const mockExtension = 'file://test-extension.js';
+    let capturedContext: any;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'afterEach') {
+        capturedContext = context;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { var1: 'value1', sessionId: 'vars-session-456' },
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedContext).toBeDefined();
+    expect(capturedContext.result.metadata.sessionId).toBe('vars-session-456');
+  });
+
+  it('should prioritize response sessionId over vars sessionId', async () => {
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        sessionId: 'response-session-priority',
+      }),
+    };
+
+    const mockExtension = 'file://test-extension.js';
+    let capturedContext: any;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'afterEach') {
+        capturedContext = context;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { var1: 'value1', sessionId: 'vars-session-ignored' },
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedContext).toBeDefined();
+    expect(capturedContext.result.metadata.sessionId).toBe('response-session-priority');
+    expect(capturedContext.result.metadata.sessionId).not.toBe('vars-session-ignored');
+  });
+
+  it('should handle empty sessionIds array', async () => {
+    const mockApiProvider = {
+      id: () => 'test-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+      }),
+    };
+
+    const mockExtension = 'file://test-extension.js';
+    let capturedContext: any;
+
+    const mockedRunExtensionHook = vi.mocked(runExtensionHook);
+    mockedRunExtensionHook.mockImplementation(async (_extensions, hookName, context) => {
+      if (hookName === 'afterEach') {
+        capturedContext = context;
+      }
+      return context;
+    });
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          vars: { var1: 'value1' },
+          metadata: {
+            sessionIds: [],
+          },
+        },
+      ],
+      extensions: [mockExtension],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+
+    expect(capturedContext).toBeDefined();
+    expect(capturedContext.result.metadata.sessionIds).toEqual([]);
+  });
+});

test/evaluator/setup.ts

@@ -0,0 +1,238 @@
+import { vi } from 'vitest';
+
+const exactTransformHandlers = new Map<string, (input: any) => any>([
+  ['output + " postprocessed"', (input) => input + ' postprocessed'],
+  ['JSON.parse(output).value', parseJsonValueTransform],
+  ['`Transformed: ${output}`', (input) => `Transformed: ${input}`],
+  ['`ProviderTransformed: ${output}`', (input) => `ProviderTransformed: ${input}`],
+  ['`Provider: ${output}`', (input) => `Provider: ${input}`],
+  ['`Test: ${output}`', (input) => `Test: ${input}`],
+  ['"testTransformed " + output', (input) => 'testTransformed ' + input],
+  ['"defaultTestTransformed " + output', (input) => 'defaultTestTransformed ' + input],
+  ['"Test: " + output', (input) => 'Test: ' + input],
+  ['"Provider: " + output', (input) => 'Provider: ' + input],
+  ['"Transform: " + output', (input) => 'Transform: ' + input],
+  ['output + "-provider-test"', (input) => input + '-provider-test'],
+  ['output + "-provider"', (input) => input + '-provider'],
+  ['output + "-test"', (input) => input + '-test'],
+  ['`Transform: ${output}`', (input) => `Transform: ${input}`],
+  ['`Postprocess: ${output}`', (input) => `Postprocess: ${input}`],
+]);
+
+function parseJsonValueTransform(input: any) {
+  try {
+    return JSON.parse(input).value;
+  } catch {
+    return input;
+  }
+}
+
+function mockTransformVars(code: string, input: any, context?: any) {
+  if (code.includes('vars.transformed = true')) {
+    return { ...input, transformed: true };
+  }
+  if (code.includes('vars.defaultTransform = true')) {
+    return { ...input, defaultTransform: true };
+  }
+  if (code.includes('{ ...vars') && code.includes('toUpperCase()')) {
+    return { ...input, name: input.name?.toUpperCase() };
+  }
+  if (code.includes('{ ...vars') && code.includes('vars.age + 5')) {
+    return { ...input, age: (input.age ?? 0) + 5 };
+  }
+  if (code.includes('return {') && code.includes('context.uuid')) {
+    return {
+      ...input,
+      id: context?.uuid || 'mock-uuid',
+      hasPrompt: Boolean(context?.prompt),
+    };
+  }
+  if (code.includes('test2UpperCase: vars.test2.toUpperCase()')) {
+    return { ...input, test2UpperCase: input.test2?.toUpperCase() };
+  }
+}
+
+function mockMetadataTransform(code: string, input: any, context?: any) {
+  if (!code.includes('context?.metadata')) {
+    return undefined;
+  }
+  if (code.includes('Output:') && context?.metadata) {
+    return `Output: ${input}, Metadata: ${JSON.stringify(context.metadata)}`;
+  }
+  if (code.includes('Has metadata') && context?.metadata) {
+    return `Has metadata: ${input}`;
+  }
+  if (code.includes('No metadata') && !context?.metadata) {
+    return `No metadata: ${input}`;
+  }
+  if (
+    code.includes('Empty metadata') &&
+    context?.metadata &&
+    Object.keys(context.metadata).length === 0
+  ) {
+    return `Empty metadata: ${input}`;
+  }
+  if (code.includes('All context') && context?.vars && context?.prompt && context?.metadata) {
+    return `All context: ${input}`;
+  }
+  if (
+    code.includes('Missing context') &&
+    !(context?.vars && context?.prompt && context?.metadata)
+  ) {
+    return `Missing context: ${input}`;
+  }
+}
+
+async function mockTransform(code: unknown, input: any, context?: any) {
+  if (typeof code !== 'string') {
+    return input;
+  }
+
+  const exactHandler = exactTransformHandlers.get(code);
+  if (exactHandler) {
+    return exactHandler(input);
+  }
+
+  return (
+    mockTransformVars(code, input, context) ?? mockMetadataTransform(code, input, context) ?? input
+  );
+}
+
+vi.mock('../../src/util/transform', () => ({
+  TransformInputType: {
+    OUTPUT: 'output',
+    VARS: 'vars',
+  },
+  // Provide a process shim for ESM compatibility in inline JavaScript code
+  getProcessShim: vi.fn().mockReturnValue(process),
+  transform: vi.fn().mockImplementation(mockTransform),
+}));
+
+vi.mock('../../src/util/fileReference', async () => {
+  const actual = await vi.importActual<typeof import('../../src/util/fileReference')>(
+    '../../src/util/fileReference',
+  );
+  return {
+    ...actual,
+    processConfigFileReferences: vi.fn().mockImplementation(async (config) => {
+      if (
+        typeof config === 'object' &&
+        config !== null &&
+        config.tests &&
+        Array.isArray(config.tests)
+      ) {
+        const result = {
+          ...config,
+          tests: config.tests.map((test: any) => {
+            return {
+              ...test,
+              vars:
+                test.vars.var1 === 'file://test/fixtures/test_file.txt'
+                  ? {
+                      var1: '<h1>Sample Report</h1><p>This is a test report with some data for the year 2023.</p>',
+                    }
+                  : test.vars,
+            };
+          }),
+        };
+        return result;
+      }
+      return config;
+    }),
+  };
+});
+
+vi.mock('proxy-agent', () => ({
+  ProxyAgent: vi.fn().mockImplementation(() => ({})),
+}));
+vi.mock('glob', () => {
+  const globSync = vi.fn().mockImplementation((pattern) => {
+    if (pattern.includes('test/fixtures/test_file.txt')) {
+      return [pattern];
+    }
+    return [];
+  });
+  const hasMagic = vi.fn((pattern: string | string[]) => {
+    const p = Array.isArray(pattern) ? pattern.join('') : pattern;
+    return p.includes('*') || p.includes('?') || p.includes('[') || p.includes('{');
+  });
+  const glob = Object.assign(vi.fn(), { globSync, hasMagic, sync: globSync });
+  return {
+    default: { globSync, hasMagic },
+    glob,
+    globSync,
+    hasMagic,
+  };
+});
+
+vi.mock('../../src/esm', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    importModule: vi.fn(),
+  };
+});
+
+vi.mock('../../src/evaluatorHelpers', async () => {
+  const actual = await vi.importActual<typeof import('../../src/evaluatorHelpers')>(
+    '../../src/evaluatorHelpers',
+  );
+  return {
+    ...actual,
+    runExtensionHook: vi.fn().mockImplementation((_extensions, _hookName, context) => context),
+  };
+});
+
+vi.mock('../../src/cliState', () => ({
+  __esModule: true,
+  default: {
+    resume: false,
+    basePath: '',
+    webUI: false,
+  },
+}));
+
+vi.mock('../../src/models/prompt', () => ({
+  generateIdFromPrompt: vi.fn((prompt) => `prompt-${prompt.label || 'default'}`),
+}));
+
+vi.mock('../../src/util/time', async () => {
+  const actual = await vi.importActual<typeof import('../../src/util/time')>('../../src/util/time');
+  return {
+    ...actual,
+    sleep: vi.fn(),
+  };
+});
+
+vi.mock('../../src/util/fileExtensions', async () => {
+  const actual = await vi.importActual<typeof import('../../src/util/fileExtensions')>(
+    '../../src/util/fileExtensions',
+  );
+  return {
+    ...actual,
+    isImageFile: vi
+      .fn()
+      .mockImplementation((filePath) => filePath.endsWith('.jpg') || filePath.endsWith('.png')),
+    isVideoFile: vi.fn().mockImplementation((filePath) => filePath.endsWith('.mp4')),
+    isAudioFile: vi.fn().mockImplementation((filePath) => filePath.endsWith('.mp3')),
+    isJavascriptFile: vi.fn().mockReturnValue(false),
+  };
+});
+
+vi.mock('../../src/util/functions/loadFunction', async () => {
+  const actual = await vi.importActual<typeof import('../../src/util/functions/loadFunction')>(
+    '../../src/util/functions/loadFunction',
+  );
+  return {
+    ...actual,
+    loadFunction: vi.fn().mockImplementation((options) => {
+      if (options.filePath.includes('scoring')) {
+        return Promise.resolve((_metrics: Record<string, number>) => ({
+          pass: true,
+          score: 0.75,
+          reason: 'Custom scoring reason',
+        }));
+      }
+      return Promise.resolve(() => {});
+    }),
+  };
+});

test/evaluator/tokenUsage.test.ts

@@ -0,0 +1,162 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate, runEval } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, mockGradingApiProviderPasses, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator token usage', () => {
+  it('should accumulate token usage correctly', async () => {
+    const mockOptions = {
+      delay: 0,
+      testIdx: 0,
+      promptIdx: 0,
+      repeatIndex: 0,
+      isRedteam: false,
+    };
+
+    const results = await runEval({
+      ...mockOptions,
+      provider: mockApiProvider,
+      prompt: { raw: 'Test prompt', label: 'test-label' },
+      test: {
+        assert: [
+          {
+            type: 'llm-rubric',
+            value: 'Test output',
+          },
+        ],
+        options: { provider: mockGradingApiProviderPasses },
+      },
+      conversations: {},
+      registers: {},
+    });
+
+    expect(results[0].tokenUsage).toEqual({
+      total: 10, // Only provider tokens, NOT assertion tokens
+      prompt: 5, // Only provider tokens
+      completion: 5, // Only provider tokens
+      cached: 0,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      numRequests: 1, // Only provider requests
+      assertions: {
+        total: 10, // Assertion tokens tracked separately
+        prompt: 5,
+        completion: 5,
+        cached: 0,
+        numRequests: 1, // Assertion requests tracked separately
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+  });
+
+  it('should NOT include assertion tokens in main token totals', async () => {
+    // Mock provider that returns fixed token usage
+    const providerWithTokens: ApiProvider = {
+      id: vi.fn().mockReturnValue('provider-with-tokens'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test response',
+        tokenUsage: {
+          total: 100,
+          prompt: 60,
+          completion: 40,
+          cached: 10,
+          numRequests: 1,
+        },
+      }),
+    };
+
+    // Mock grading provider that also returns token usage
+    const gradingProviderWithTokens: ApiProvider = {
+      id: vi.fn().mockReturnValue('grading-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          pass: true,
+          score: 1,
+          reason: 'Test passed',
+        }),
+        tokenUsage: {
+          total: 50,
+          prompt: 30,
+          completion: 20,
+          cached: 5,
+          numRequests: 1,
+        },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [providerWithTokens],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'llm-rubric',
+              value: 'Output should be valid',
+              provider: gradingProviderWithTokens,
+            },
+          ],
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    // Verify main totals only include provider tokens, NOT assertion tokens
+    expect(summary.stats.tokenUsage).toEqual({
+      total: 100, // Only provider tokens
+      prompt: 60,
+      completion: 40,
+      cached: 10,
+      numRequests: 1,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+      assertions: {
+        total: 50, // Assertion tokens tracked separately
+        prompt: 30,
+        completion: 20,
+        cached: 5,
+        numRequests: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+          cacheReadInputTokens: 0,
+          cacheCreationInputTokens: 0,
+        },
+      },
+    });
+
+    // Also verify at the result level - the result should pass
+    const result = summary.results[0];
+    expect(result).toHaveProperty('success', true);
+    expect(result).toHaveProperty('score', 1);
+
+    // The main verification is at the stats level (already done above)
+    // Individual results may not always have tokenUsage populated in the summary
+  });
+});

test/evaluator/trace-integration.test.ts

@@ -0,0 +1,371 @@
+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import { nodeEvaluatorRuntime } from '../../src/node/evaluatorRuntime';
+import * as evaluatorTracing from '../../src/tracing/evaluatorTracing';
+import { getTraceStore } from '../../src/tracing/store';
+import { createMockProvider } from '../factories/provider';
+
+import type { EvaluatorRuntime } from '../../src/evaluator/runtime';
+import type Eval from '../../src/models/eval';
+import type EvalResult from '../../src/models/evalResult';
+import type { EvaluateOptions, TestSuite } from '../../src/types/index';
+
+// Mock dependencies
+vi.mock('../../src/tracing/store');
+const mockFlushOtel = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+const mockInitializeOtel = vi.hoisted(() => vi.fn());
+const mockShutdownOtel = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+
+vi.mock('../../src/tracing/otelSdk', () => ({
+  flushOtel: mockFlushOtel,
+  initializeOtel: mockInitializeOtel,
+  shutdownOtel: mockShutdownOtel,
+}));
+
+vi.mock('../../src/tracing/otlpReceiver', () => ({
+  startOTLPReceiver: vi.fn(),
+  stopOTLPReceiver: vi.fn(),
+}));
+
+// Mock evaluatorTracing module
+vi.mock('../../src/tracing/evaluatorTracing', () => ({
+  generateTraceId: vi.fn(() => 'abcdef1234567890abcdef1234567890'),
+  generateSpanId: vi.fn(() => '0123456789abcdef'),
+  generateTraceparent: vi.fn((traceId, spanId) => `00-${traceId}-${spanId}-01`),
+  generateTraceContextIfNeeded: vi.fn(),
+  startOtlpReceiverIfNeeded: vi.fn(),
+  stopOtlpReceiverIfNeeded: vi.fn(),
+  isOtlpReceiverStarted: vi.fn(() => false),
+  isTracingEnabled: vi.fn((test) => test.metadata?.tracingEnabled === true),
+}));
+
+describe('evaluator trace integration', () => {
+  const mockTraceStore = {
+    createTrace: vi.fn(),
+    getTrace: vi.fn(),
+  };
+
+  const mockEval = {
+    id: 'test-eval-id',
+    addResult: vi.fn(),
+    addPrompts: vi.fn(),
+    fetchResultsByTestIdx: vi.fn(),
+    setVars: vi.fn(),
+    setDurationMs: vi.fn(),
+    results: [],
+    prompts: [],
+    persisted: false,
+    config: {
+      outputPath: undefined,
+    },
+  } as unknown as Eval;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    (getTraceStore as Mock).mockReturnValue(mockTraceStore);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should pass traceId through to assertions when tracing is enabled', async () => {
+    // Mock trace creation and retrieval
+    const testTraceId = 'abcdef1234567890abcdef1234567890';
+    mockTraceStore.createTrace.mockResolvedValue(undefined);
+    mockTraceStore.getTrace.mockResolvedValue({
+      traceId: testTraceId,
+      spans: [
+        {
+          spanId: 'test-span',
+          name: 'test.operation',
+          startTime: 1000,
+          endTime: 2000,
+        },
+      ],
+    });
+
+    // Mock generateTraceContextIfNeeded
+    vi.mocked(evaluatorTracing.generateTraceContextIfNeeded).mockResolvedValue({
+      traceparent: `00-${testTraceId}-0123456789abcdef-01`,
+      evaluationId: 'test-eval-id',
+      testCaseId: 'test-case-id',
+    });
+
+    const testSuite: TestSuite = {
+      providers: [
+        createMockProvider({
+          id: 'mock-provider',
+          response: { output: 'Test response', tokenUsage: {} },
+        }),
+      ],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [
+        {
+          vars: { input: 'test' },
+          metadata: {
+            tracingEnabled: true,
+            evaluationId: 'test-eval-id',
+          },
+          assert: [
+            {
+              type: 'javascript',
+              value: `
+                // Verify trace data is available
+                if (!context.trace) return false;
+                return context.trace.spans.length > 0 &&
+                       context.trace.spans[0].name === 'test.operation';
+              `,
+            },
+          ],
+        },
+      ],
+      tracing: {
+        enabled: true,
+        otlp: {
+          http: {
+            enabled: true,
+            port: 4318,
+            host: '0.0.0.0',
+            acceptFormats: ['protobuf'],
+          },
+        },
+      },
+    };
+
+    const options: EvaluateOptions = {
+      maxConcurrency: 1,
+    };
+
+    // Run evaluation
+    await evaluate(testSuite, mockEval, options);
+
+    // Verify trace context was generated
+    expect(evaluatorTracing.generateTraceContextIfNeeded).toHaveBeenCalled();
+    expect(evaluatorTracing.startOtlpReceiverIfNeeded).toHaveBeenCalledWith(
+      testSuite,
+      'test-eval-id',
+    );
+
+    // Verify trace was fetched for assertion
+    expect(mockTraceStore.getTrace).toHaveBeenCalledWith(testTraceId, {
+      sanitizeAttributes: false,
+    });
+    expect(mockFlushOtel).toHaveBeenCalled();
+    expect(mockShutdownOtel).toHaveBeenCalledOnce();
+
+    // Verify result was added with passing assertion
+    expect(mockEval.addResult).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: true,
+        score: 1,
+      }),
+    );
+  });
+
+  it('closes writers but skips OTEL shutdown when initialization fails', async () => {
+    const writer = {
+      write: vi.fn().mockResolvedValue(undefined),
+      close: vi.fn().mockResolvedValue(undefined),
+    };
+    const runtime: EvaluatorRuntime<Eval, EvalResult> = {
+      createEvaluationStore: nodeEvaluatorRuntime.createEvaluationStore,
+      createResultWriters: vi.fn().mockReturnValue([writer]),
+    };
+    mockInitializeOtel.mockImplementationOnce(() => {
+      throw new Error('otel unavailable');
+    });
+
+    await expect(
+      evaluate(
+        {
+          providers: [],
+          prompts: [],
+          tests: [],
+          tracing: { enabled: true },
+        },
+        mockEval,
+        {},
+        runtime,
+      ),
+    ).rejects.toThrow('otel unavailable');
+
+    expect(writer.close).toHaveBeenCalledOnce();
+    expect(mockFlushOtel).not.toHaveBeenCalled();
+    expect(mockShutdownOtel).not.toHaveBeenCalled();
+  });
+
+  it('flushes and shuts down OTEL after successful tracing initialization', async () => {
+    await evaluate(
+      {
+        providers: [
+          createMockProvider({
+            id: 'mock-provider',
+            response: { output: 'Test response', tokenUsage: {} },
+          }),
+        ],
+        prompts: [{ raw: 'Test prompt', label: 'test' }],
+        tests: [{}],
+        tracing: { enabled: true },
+      },
+      mockEval,
+      {},
+    );
+
+    expect(mockInitializeOtel).toHaveBeenCalledOnce();
+    expect(mockFlushOtel).toHaveBeenCalledOnce();
+    expect(mockShutdownOtel).toHaveBeenCalledOnce();
+  });
+
+  it('should handle assertions gracefully when tracing is disabled', async () => {
+    // Mock generateTraceContextIfNeeded to return null when tracing is disabled
+    vi.mocked(evaluatorTracing.generateTraceContextIfNeeded).mockResolvedValue(null);
+
+    const testSuite: TestSuite = {
+      providers: [
+        createMockProvider({
+          id: 'mock-provider',
+          response: { output: 'Test response', tokenUsage: {} },
+        }),
+      ],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [
+        {
+          vars: { input: 'test' },
+          // No tracingEnabled in metadata
+          assert: [
+            {
+              type: 'javascript',
+              value: `
+                // Should pass when trace is undefined
+                return context.trace === undefined && output === 'Test response';
+              `,
+            },
+          ],
+        },
+      ],
+      // Tracing not enabled in test suite
+    };
+
+    const options: EvaluateOptions = {
+      maxConcurrency: 1,
+    };
+
+    // Run evaluation
+    await evaluate(testSuite, mockEval, options);
+
+    // Verify trace was NOT created or fetched
+    expect(mockTraceStore.createTrace).not.toHaveBeenCalled();
+    expect(mockTraceStore.getTrace).not.toHaveBeenCalled();
+    expect(mockFlushOtel).not.toHaveBeenCalled();
+    expect(mockShutdownOtel).not.toHaveBeenCalled();
+
+    // Verify result was added with passing assertion
+    expect(mockEval.addResult).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: true,
+        score: 1,
+      }),
+    );
+  });
+
+  it('should extract traceId correctly from traceparent header', async () => {
+    const testTraceId = '0af7651916cd43dd8448eb211c80319c';
+    const testSpanId = 'b7ad6b7169203331';
+
+    // Mock the trace context generation
+    vi.mocked(evaluatorTracing.generateTraceContextIfNeeded).mockResolvedValue({
+      traceparent: `00-${testTraceId}-${testSpanId}-01`,
+      evaluationId: 'test-eval-id',
+      testCaseId: 'test-case-id',
+    });
+
+    mockTraceStore.createTrace.mockResolvedValue(undefined);
+    mockTraceStore.getTrace.mockResolvedValue({
+      traceId: testTraceId,
+      spans: [
+        {
+          spanId: 'test-span',
+          name: 'extracted.correctly',
+          startTime: 1000,
+          endTime: 2000,
+        },
+      ],
+    });
+
+    const testSuite: TestSuite = {
+      providers: [
+        createMockProvider({
+          id: 'mock-provider',
+          response: { output: 'Test response', tokenUsage: {} },
+        }),
+      ],
+      prompts: [{ raw: 'Test prompt', label: 'test' }],
+      tests: [
+        {
+          vars: { input: 'test' },
+          metadata: {
+            tracingEnabled: true,
+            evaluationId: 'test-eval-id',
+          },
+          assert: [
+            {
+              type: 'javascript',
+              value: `
+                // Verify the extracted traceId matches
+                return context.trace && context.trace.traceId === '${testTraceId}';
+              `,
+            },
+          ],
+        },
+      ],
+    };
+
+    const options: EvaluateOptions = {
+      maxConcurrency: 1,
+    };
+
+    // Run evaluation
+    await evaluate(testSuite, mockEval, options);
+
+    // Verify trace was fetched with the correct traceId
+    expect(mockTraceStore.getTrace).toHaveBeenCalledWith(testTraceId, {
+      sanitizeAttributes: false,
+    });
+
+    // Verify result was added with passing assertion
+    expect(mockEval.addResult).toHaveBeenCalledWith(
+      expect.objectContaining({
+        success: true,
+      }),
+    );
+  });
+
+  it('should still run evaluator cleanup when receiver startup is fatal', async () => {
+    vi.mocked(evaluatorTracing.startOtlpReceiverIfNeeded).mockRejectedValueOnce(
+      new Error('receiver failed'),
+    );
+
+    const testSuite: TestSuite = {
+      providers: [],
+      prompts: [],
+      tests: [],
+      tracing: {
+        enabled: true,
+        failOnReceiverStartFailure: true,
+        otlp: {
+          http: {
+            enabled: true,
+            port: 4318,
+            host: '127.0.0.1',
+          },
+        },
+      },
+    };
+
+    await expect(evaluate(testSuite, mockEval, {})).rejects.toThrow('receiver failed');
+    expect(evaluatorTracing.stopOtlpReceiverIfNeeded).toHaveBeenCalled();
+    expect(mockFlushOtel).not.toHaveBeenCalled();
+    expect(mockShutdownOtel).not.toHaveBeenCalled();
+  });
+});

test/evaluator/traceLinkage.test.ts

@@ -0,0 +1,93 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getTraceId, getTraceLinkage } from '../../src/evaluator';
+import logger from '../../src/logger';
+
+const TRACE_ID = 'a'.repeat(32);
+const SPAN_ID = 'b'.repeat(16);
+// Well-formed W3C v00 traceparent (4 parts) and a forward-compatible 5+ part variant.
+const TP_V00 = `00-${TRACE_ID}-${SPAN_ID}-01`;
+const TP_FUTURE = `01-${TRACE_ID}-${SPAN_ID}-01-extra-field`;
+
+describe('getTraceId', () => {
+  let warnSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => logger);
+  });
+
+  afterEach(() => {
+    warnSpy.mockRestore();
+  });
+
+  it('returns undefined when no trace context or traceparent is present', () => {
+    expect(getTraceId(undefined)).toBeUndefined();
+    expect(getTraceId(null)).toBeUndefined();
+    expect(getTraceId({})).toBeUndefined();
+    expect(getTraceId({ evaluationId: 'eval-x' })).toBeUndefined();
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('extracts the trace id from a well-formed v00 traceparent (4 parts)', () => {
+    expect(getTraceId({ traceparent: TP_V00 })).toBe(TRACE_ID);
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('accepts forward-compatible traceparents with more than 4 parts', () => {
+    // W3C allows future versions to append fields after flags; still read trace-id at index 1.
+    expect(getTraceId({ traceparent: TP_FUTURE })).toBe(TRACE_ID);
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    ['too few fields', `00-${TRACE_ID}-01`],
+    ['unparseable value', 'garbage'],
+    ['extra fields on version 00', `${TP_V00}-extra-field`],
+    ['forbidden ff version', `ff-${TRACE_ID}-${SPAN_ID}-01`],
+    ['empty trace id', `00--${SPAN_ID}-01`],
+    ['all-zero trace id', `00-${'0'.repeat(32)}-${SPAN_ID}-01`],
+    ['uppercase trace id', `00-${TRACE_ID.toUpperCase()}-${SPAN_ID}-01`],
+    ['all-zero parent id', `00-${TRACE_ID}-${'0'.repeat(16)}-01`],
+    ['invalid flags', `00-${TRACE_ID}-${SPAN_ID}-0g`],
+  ])('drops linkage and warns for %s', (_description, traceparent) => {
+    expect(getTraceId({ traceparent })).toBeUndefined();
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+    expect(warnSpy.mock.calls[0][0]).toContain('Malformed traceparent');
+  });
+});
+
+describe('getTraceLinkage', () => {
+  let warnSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => logger);
+  });
+
+  afterEach(() => {
+    warnSpy.mockRestore();
+  });
+
+  it('returns an empty object when there is no trace context', () => {
+    expect(getTraceLinkage(undefined)).toEqual({});
+    expect(getTraceLinkage(null, 'eval-fallback')).toEqual({});
+  });
+
+  it('pairs the trace id and evaluation id from the context', () => {
+    const linkage = getTraceLinkage({ traceparent: TP_V00, evaluationId: 'eval-1' });
+    expect(linkage).toEqual({ traceId: TRACE_ID, evaluationId: 'eval-1' });
+  });
+
+  it('falls back to the eval id when the context lacks an evaluation id', () => {
+    const linkage = getTraceLinkage({ traceparent: TP_V00 }, 'eval-fallback');
+    expect(linkage).toEqual({ traceId: TRACE_ID, evaluationId: 'eval-fallback' });
+  });
+
+  it('emits evaluationId without traceId when the traceparent is malformed', () => {
+    // A trace context exists (the row was traced) but the trace id could not be parsed.
+    const linkage = getTraceLinkage({ traceparent: 'malformed', evaluationId: 'eval-2' });
+    expect(linkage).toEqual({ evaluationId: 'eval-2' });
+  });
+
+  it('omits both fields when a traced context has neither a parseable id nor an eval id', () => {
+    expect(getTraceLinkage({ traceparent: 'malformed' })).toEqual({});
+  });
+});

test/evaluator/transforms.test.ts

@@ -0,0 +1,527 @@
+import './setup';
+
+import { randomUUID } from 'crypto';
+
+import { expect, it, vi } from 'vitest';
+import { evaluate } from '../../src/evaluator';
+import Eval from '../../src/models/eval';
+import { type ApiProvider, type TestSuite } from '../../src/types/index';
+import { mockApiProvider, toPrompt } from './helpers';
+import { describeEvaluator } from './lifecycle';
+
+describeEvaluator('evaluator transforms', () => {
+  it('evaluate with transform option - default test', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      defaultTest: {
+        options: {
+          transform: 'output + " postprocessed"',
+        },
+      },
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+  });
+
+  it('evaluate with transform option - single test', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output postprocessed',
+            },
+          ],
+          options: {
+            transform: 'output + " postprocessed"',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+  });
+
+  it('evaluate with transform option - json provider', async () => {
+    const mockApiJsonProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-json'),
+      callApi: vi.fn().mockResolvedValue({
+        output: '{"output": "testing", "value": 123}',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiJsonProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: '123',
+            },
+          ],
+          options: {
+            transform: `JSON.parse(output).value`,
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiJsonProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe(123);
+  });
+
+  it('evaluate with provider transform', async () => {
+    const mockApiProviderWithTransform: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-transform'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Original output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+      transform: '`Transformed: ${output}`',
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithTransform],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Transformed: Original output',
+            },
+          ],
+          options: {}, // No test transform, relying on provider's transform
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Transformed: Original output');
+  });
+
+  it('evaluate with vars transform', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Hello {{ name }}, your age is {{ age }}')],
+      tests: [
+        {
+          vars: { name: 'Alice', age: 30 },
+        },
+        {
+          vars: { name: 'Bob', age: 25 },
+          options: {
+            transformVars: '{ ...vars, age: vars.age + 5 }',
+          },
+        },
+      ],
+      defaultTest: {
+        options: {
+          transformVars: '{ ...vars, name: vars.name.toUpperCase() }',
+        },
+      },
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary).toEqual(
+      expect.objectContaining({
+        stats: expect.objectContaining({
+          successes: 2,
+          failures: 0,
+        }),
+        results: expect.arrayContaining([
+          expect.objectContaining({
+            prompt: expect.objectContaining({
+              raw: 'Hello ALICE, your age is 30',
+              label: 'Hello {{ name }}, your age is {{ age }}',
+            }),
+            response: expect.objectContaining({
+              output: 'Test output',
+            }),
+          }),
+          expect.objectContaining({
+            // NOTE: test overrides defaultTest transform. Bob not BOB
+            prompt: expect.objectContaining({
+              raw: 'Hello Bob, your age is 30',
+            }),
+            response: expect.objectContaining({
+              output: 'Test output',
+            }),
+            vars: {
+              name: 'Bob',
+              age: 30,
+            },
+          }),
+        ]),
+      }),
+    );
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+  });
+
+  it('evaluate with context in vars transform in defaultTest', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Hello {{ name }}, your age is {{ age }}')],
+      defaultTest: {
+        options: {
+          transformVars: `return {
+                ...vars,
+                // Test that context.uuid is available
+                id: context.uuid,
+                // Test that context.prompt is available but empty
+                hasPrompt: Boolean(context.prompt)
+              }`,
+        },
+      },
+      tests: [
+        {
+          vars: {
+            name: 'Alice',
+            age: 25,
+          },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary).toEqual(
+      expect.objectContaining({
+        stats: expect.objectContaining({
+          successes: 1,
+          failures: 0,
+        }),
+        results: expect.arrayContaining([
+          expect.objectContaining({
+            vars: expect.objectContaining({
+              id: expect.stringMatching(
+                /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,
+              ),
+              hasPrompt: true,
+            }),
+          }),
+        ]),
+      }),
+    );
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('evaluate with provider transform and test transform', async () => {
+    const mockApiProviderWithTransform: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-transform'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Original output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+      transform: '`ProviderTransformed: ${output}`',
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithTransform],
+      prompts: [toPrompt('Test prompt')],
+      defaultTest: {
+        options: {
+          // overridden by the test transform
+          transform: '"defaultTestTransformed " + output',
+        },
+      },
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              // Order of transforms: 1. Provider transform 2. Test transform (or defaultTest transform, if test transform unset)
+              value: 'testTransformed ProviderTransformed: Original output',
+            },
+          ],
+          // This transform overrides the defaultTest transform
+          options: { transform: '"testTransformed " + output' },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary).toEqual(
+      expect.objectContaining({
+        stats: expect.objectContaining({
+          successes: 1,
+          failures: 0,
+        }),
+        results: expect.arrayContaining([
+          expect.objectContaining({
+            response: expect.objectContaining({
+              output: 'testTransformed ProviderTransformed: Original output',
+            }),
+          }),
+        ]),
+      }),
+    );
+
+    expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('evaluate with multiple transforms', async () => {
+    const mockApiProviderWithTransform: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-transform'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Original output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+      transform: '`Provider: ${output}`',
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithTransform],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test: Provider: Original output',
+            },
+          ],
+          options: {
+            transform: '`Test: ${output}`',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Test: Provider: Original output');
+  });
+
+  it('evaluate with provider transform and test postprocess (deprecated)', async () => {
+    const mockApiProviderWithTransform: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-transform'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Original output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+      transform: '`Provider: ${output}`',
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithTransform],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Postprocess: Provider: Original output',
+            },
+          ],
+          options: {
+            postprocess: '`Postprocess: ${output}`',
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary).toMatchObject({
+      stats: expect.objectContaining({
+        successes: 1,
+        failures: 0,
+      }),
+    });
+    expect(summary.results[0].response?.output).toBe('Postprocess: Provider: Original output');
+
+    expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('evaluate with provider transform, test transform, and test postprocess (deprecated)', async () => {
+    const mockApiProviderWithTransform: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-transform'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Original output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+      transform: '`Provider: ${output}`',
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderWithTransform],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Transform: Provider: Original output',
+            },
+          ],
+          options: {
+            transform: '`Transform: ${output}`',
+            postprocess: '`Postprocess: ${output}`', // This should be ignored
+          },
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+    expect(summary).toMatchObject({
+      stats: expect.objectContaining({
+        successes: 1,
+        failures: 0,
+      }),
+      results: expect.arrayContaining([
+        expect.objectContaining({
+          response: expect.objectContaining({
+            output: 'Transform: Provider: Original output',
+          }),
+        }),
+      ]),
+    });
+    expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('evaluate with no output', async () => {
+    const mockApiProviderNoOutput: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-no-output'),
+      callApi: vi.fn().mockResolvedValue({
+        output: null,
+        tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderNoOutput],
+      prompts: [toPrompt('Test prompt')],
+      tests: [],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(0);
+    expect(summary.stats.failures).toBe(1);
+    expect(summary.results[0].error).toBe('No output');
+    expect(summary.results[0].success).toBe(false);
+    expect(summary.results[0].score).toBe(0);
+    expect(mockApiProviderNoOutput.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('evaluate with false output', async () => {
+    const mockApiProviderNoOutput: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider-no-output'),
+      callApi: vi.fn().mockResolvedValue({
+        output: false,
+        tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProviderNoOutput],
+      prompts: [toPrompt('Test prompt')],
+      tests: [],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].success).toBe(true);
+    expect(summary.results[0].score).toBe(1);
+    expect(mockApiProviderNoOutput.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('merges defaultTest.vars before applying transformVars', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ test1 }} {{ test2 }} {{ test2UpperCase }}')],
+      defaultTest: {
+        vars: {
+          test2: 'bar',
+        },
+        options: {
+          transformVars: `
+              return {
+                ...vars,
+                test2UpperCase: vars.test2.toUpperCase()
+              };
+            `,
+        },
+      },
+      tests: [
+        {
+          vars: {
+            test1: 'foo',
+          },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+    await evaluate(testSuite, evalRecord, {});
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+
+    // Check that vars were merged correctly and transform was applied
+    expect(summary.results[0].vars).toEqual({
+      test1: 'foo',
+      test2: 'bar',
+      test2UpperCase: 'BAR',
+    });
+
+    // Verify the prompt was rendered with all variables
+    expect(summary.results[0].prompt.raw).toBe('Test prompt foo bar BAR');
+  });
+});

test/evaluator/utils.test.ts

@@ -0,0 +1,226 @@
+import './setup';
+
+import { globSync } from 'glob';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import {
+  formatVarsForDisplay,
+  generateVarCombinations,
+  isAllowedPrompt,
+} from '../../src/evaluator';
+import { type Prompt } from '../../src/types/index';
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe('generateVarCombinations', () => {
+  it('should generate combinations for simple variables', () => {
+    const vars = { language: 'English', greeting: 'Hello' };
+    const expected = [{ language: 'English', greeting: 'Hello' }];
+    expect(generateVarCombinations(vars)).toEqual(expected);
+  });
+
+  it('should generate combinations for array variables', () => {
+    const vars = { language: ['English', 'French'], greeting: 'Hello' };
+    const expected = [
+      { language: 'English', greeting: 'Hello' },
+      { language: 'French', greeting: 'Hello' },
+    ];
+    expect(generateVarCombinations(vars)).toEqual(expected);
+  });
+
+  it('should handle file paths and expand them into combinations', () => {
+    const vars = { language: 'English', greeting: 'file:///path/to/greetings/*.txt' };
+    vi.mocked(globSync).mockReturnValue(['greeting1.txt', 'greeting2.txt']);
+    const expected = [
+      { language: 'English', greeting: 'file://greeting1.txt' },
+      { language: 'English', greeting: 'file://greeting2.txt' },
+    ];
+    expect(generateVarCombinations(vars)).toEqual(expected);
+  });
+
+  it('should correctly handle nested array variables', () => {
+    const vars = {
+      options: [
+        ['opt1', 'opt2'],
+        ['opt3', 'opt4'],
+      ],
+    };
+    const expected = [
+      {
+        options: [
+          ['opt1', 'opt2'],
+          ['opt3', 'opt4'],
+        ],
+      },
+    ];
+    expect(generateVarCombinations(vars)).toEqual(expected);
+  });
+
+  it('should return an empty array for empty input', () => {
+    const vars = {};
+    const expected = [{}];
+    expect(generateVarCombinations(vars)).toEqual(expected);
+  });
+});
+
+describe('isAllowedPrompt', () => {
+  const prompt1: Prompt = {
+    label: 'prompt1',
+    raw: '',
+  };
+  const prompt2: Prompt = {
+    label: 'group1:prompt2',
+    raw: '',
+  };
+  const prompt3: Prompt = {
+    label: 'group2:prompt3',
+    raw: '',
+  };
+
+  it('should return true if allowedPrompts is undefined', () => {
+    expect(isAllowedPrompt(prompt1, undefined)).toBe(true);
+  });
+
+  it('should return true if allowedPrompts includes the prompt label', () => {
+    expect(isAllowedPrompt(prompt1, ['prompt1', 'prompt2'])).toBe(true);
+  });
+
+  it('should return true if allowedPrompts includes a label that matches the start of the prompt label followed by a colon', () => {
+    expect(isAllowedPrompt(prompt2, ['group1'])).toBe(true);
+  });
+
+  it('should return true if allowedPrompts includes a wildcard prefix', () => {
+    expect(isAllowedPrompt(prompt2, ['group1:*'])).toBe(true);
+  });
+
+  it('should return false if a wildcard prefix does not match the prompt label', () => {
+    expect(isAllowedPrompt(prompt3, ['group1:*'])).toBe(false);
+  });
+
+  it('should return false if allowedPrompts does not include the prompt label or any matching start label with a colon', () => {
+    expect(isAllowedPrompt(prompt3, ['group1', 'prompt2'])).toBe(false);
+  });
+
+  it('should return false if allowedPrompts is an empty array', () => {
+    expect(isAllowedPrompt(prompt1, [])).toBe(false);
+  });
+});
+
+describe('formatVarsForDisplay', () => {
+  it('should return empty string for empty or undefined vars', () => {
+    expect(formatVarsForDisplay({}, 50)).toBe('');
+    expect(formatVarsForDisplay(undefined, 50)).toBe('');
+    expect(formatVarsForDisplay(null as any, 50)).toBe('');
+  });
+
+  it('should format simple variables correctly', () => {
+    const vars = { name: 'John', age: 25, city: 'NYC' };
+    const result = formatVarsForDisplay(vars, 50);
+
+    expect(result).toBe('name=John age=25 city=NYC');
+  });
+
+  it('should handle different variable types', () => {
+    const vars = {
+      string: 'hello',
+      number: 42,
+      boolean: true,
+      nullValue: null,
+      undefinedValue: undefined,
+      object: { nested: 'value' },
+      array: [1, 2, 3],
+    };
+
+    const result = formatVarsForDisplay(vars, 200);
+
+    expect(result).toContain('string=hello');
+    expect(result).toContain('number=42');
+    expect(result).toContain('boolean=true');
+    expect(result).toContain('nullValue=null');
+    expect(result).toContain('undefinedValue=undefined');
+    expect(result).toContain('object=[object Object]');
+    expect(result).toContain('array=1,2,3');
+  });
+
+  it('should truncate individual values to prevent memory issues', () => {
+    const bigValue = 'x'.repeat(200);
+    const vars = { bigVar: bigValue };
+
+    const result = formatVarsForDisplay(vars, 200);
+
+    // Should truncate the value to 100 chars
+    expect(result).toBe(`bigVar=${'x'.repeat(100)}`);
+    expect(result.length).toBeLessThanOrEqual(200);
+  });
+
+  it('should handle extremely large vars without crashing', () => {
+    // This would have caused RangeError before the fix
+    const megaString = 'x'.repeat(500 * 1024); // 500KB string (reduced from 5MB to prevent SIGSEGV on macOS/Node24)
+    const vars = {
+      mega1: megaString,
+      mega2: megaString,
+      small: 'normal',
+    };
+
+    expect(() => formatVarsForDisplay(vars, 50)).not.toThrow();
+
+    const result = formatVarsForDisplay(vars, 50);
+    expect(typeof result).toBe('string');
+    expect(result.length).toBeLessThanOrEqual(50);
+  });
+
+  it('should truncate final result to maxLength', () => {
+    const vars = {
+      var1: 'value1',
+      var2: 'value2',
+      var3: 'value3',
+      var4: 'value4',
+    };
+
+    const result = formatVarsForDisplay(vars, 20);
+
+    expect(result.length).toBeLessThanOrEqual(20);
+    expect(result).toBe('var1=value1 var2=val');
+  });
+
+  it('should replace newlines with spaces', () => {
+    const vars = {
+      multiline: 'line1\nline2\nline3',
+    };
+
+    const result = formatVarsForDisplay(vars, 100);
+
+    expect(result).toBe('multiline=line1 line2 line3');
+    expect(result).not.toContain('\n');
+  });
+
+  it('should return fallback message on any error', () => {
+    // Create a problematic object that might throw during String() conversion
+    const problematicVars = {
+      badProp: {
+        toString() {
+          throw new Error('Cannot convert to string');
+        },
+      },
+    };
+
+    const result = formatVarsForDisplay(problematicVars, 50);
+
+    expect(result).toBe('[vars unavailable]');
+  });
+
+  it('should handle multiple variables with space distribution', () => {
+    const vars = {
+      a: 'short',
+      b: 'medium_value',
+      c: 'a_very_long_value_that_exceeds_normal_length',
+    };
+
+    const result = formatVarsForDisplay(vars, 30);
+
+    expect(result.length).toBeLessThanOrEqual(30);
+    expect(result).toContain('a=short');
+    // Should fit as much as possible within the limit
+  });
+});