chore: import upstream snapshot with attribution

test/util/eval/filterFailingBug.test.ts

@@ -0,0 +1,734 @@
+/**
+ * Test for the --filter-failing bug where test.vars gets mutated
+ * during evaluation, causing mismatches when trying to filter failing tests.
+ *
+ * Bug 1: When evaluation adds runtime vars like _conversation to test.vars,
+ * it mutates the original test object. This mutated test is stored in results.
+ * On subsequent runs with --filter-failing, freshly loaded tests don't have
+ * these runtime vars, so deepEqual comparison fails.
+ *
+ * Bug 2: Multi-turn strategy providers (GOAT, Crescendo, SIMBA) add sessionId
+ * to vars during execution, causing the same mismatch issue.
+ *
+ * Fix: We filter out known runtime vars (_conversation, sessionId) during test
+ * matching in resultIsForTestCase, and create a shallow copy of test.vars in
+ * runEval to prevent mutation by reference.
+ */
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import Eval from '../../../src/models/eval';
+import { ResultFailureReason } from '../../../src/types/index';
+import { filterTests } from '../../../src/util/eval/filterTests';
+
+import type { TestSuite } from '../../../src/types/index';
+
+vi.mock('../../../src/models/eval', () => ({
+  default: {
+    findById: vi.fn(),
+  },
+}));
+
+describe('filterTests - vars mutation bug', () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('should match tests even when stored results have additional runtime vars', async () => {
+    // Simulate a test suite as it would be loaded from config
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'test1',
+          vars: { input: 'hello', language: 'en' },
+          assert: [],
+        },
+        {
+          description: 'test2',
+          vars: { input: 'goodbye', language: 'en' },
+          assert: [],
+        },
+        {
+          description: 'test3',
+          vars: { input: 'thanks', language: 'en' },
+          assert: [],
+        },
+      ],
+    };
+
+    // Simulate stored results where test.vars was mutated with _conversation
+    // This is what happens in the bug - the runtime adds _conversation to vars
+    const mockEval = {
+      id: 'eval-123',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // This test failed and has the original vars
+            vars: { input: 'hello', language: 'en' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'test1',
+              // Bug: stored testCase.vars has _conversation added
+              vars: { input: 'hello', language: 'en', _conversation: [] },
+              assert: [],
+            },
+          },
+          {
+            // This test also failed (assertion failure, not error)
+            vars: { input: 'thanks', language: 'en' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'test3',
+              // Bug: stored testCase.vars has _conversation added
+              vars: { input: 'thanks', language: 'en', _conversation: [] },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // When filtering failing tests, it should match based on the original vars
+    // not fail because of the extra _conversation property
+    const result = await filterTests(mockTestSuite, { failing: 'eval-123' });
+
+    // Should find both failing tests (both have ASSERT failure reason)
+    expect(result).toHaveLength(2);
+    expect(result.map((t) => t.description)).toEqual(['test1', 'test3']);
+  });
+
+  it('should include both failures and errors when using --filter-failing', async () => {
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        { description: 'test1', vars: { input: 'a' }, assert: [] },
+        { description: 'test2', vars: { input: 'b' }, assert: [] },
+        { description: 'test3', vars: { input: 'c' }, assert: [] },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-errors',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { input: 'a' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT, // Actual failure
+            testCase: { description: 'test1', vars: { input: 'a' }, assert: [] },
+          },
+          {
+            vars: { input: 'b' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR, // Error
+            testCase: { description: 'test2', vars: { input: 'b' }, assert: [] },
+          },
+          {
+            vars: { input: 'c' },
+            success: true, // Success
+            testCase: { description: 'test3', vars: { input: 'c' }, assert: [] },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // --filter-failing should return both failures AND errors (all non-successful results)
+    const result = await filterTests(mockTestSuite, { failing: 'eval-errors' });
+
+    // Should find test1 (ASSERT failure) and test2 (ERROR), but not test3 (success)
+    expect(result).toHaveLength(2);
+    expect(result.map((t) => t.description).sort()).toEqual(['test1', 'test2']);
+  });
+
+  it('should exclude error results when using --filter-failing-only', async () => {
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        { description: 'test1', vars: { input: 'a' }, assert: [] },
+        { description: 'test2', vars: { input: 'b' }, assert: [] },
+        { description: 'test3', vars: { input: 'c' }, assert: [] },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-errors',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { input: 'a' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT, // Actual failure
+            testCase: { description: 'test1', vars: { input: 'a' }, assert: [] },
+          },
+          {
+            vars: { input: 'b' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR, // Error, not failure
+            testCase: { description: 'test2', vars: { input: 'b' }, assert: [] },
+          },
+          {
+            vars: { input: 'c' },
+            success: true, // Success
+            testCase: { description: 'test3', vars: { input: 'c' }, assert: [] },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // --filter-failing-only should only return actual failures, not errors
+    const result = await filterTests(mockTestSuite, { failingOnly: 'eval-errors' });
+
+    // Should only find test1 (the one with ASSERT failure), not test2 (error)
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('test1');
+  });
+
+  it('should match tests even when stored results have sessionId added by GOAT/Crescendo providers', async () => {
+    // Simulate a redteam test suite with GOAT strategy
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'goat-test-1',
+          vars: { prompt: 'test attack prompt', goal: 'test goal' },
+          assert: [],
+          metadata: { pluginId: 'harmful', strategyId: 'goat' },
+        },
+        {
+          description: 'goat-test-2',
+          vars: { prompt: 'another attack prompt', goal: 'another goal' },
+          assert: [],
+          metadata: { pluginId: 'harmful', strategyId: 'goat' },
+        },
+      ],
+    };
+
+    // Simulate stored results where vars was mutated with sessionId by GOAT provider
+    const mockEval = {
+      id: 'eval-456',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // GOAT provider added sessionId during multi-turn attack
+            vars: {
+              prompt: 'test attack prompt',
+              goal: 'test goal',
+              sessionId: 'goat-session-abc',
+            },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: {
+              description: 'goat-test-1',
+              // Stored testCase.vars has sessionId added by GOAT provider
+              vars: {
+                prompt: 'test attack prompt',
+                goal: 'test goal',
+                sessionId: 'goat-session-abc',
+              },
+              assert: [],
+              metadata: { pluginId: 'harmful', strategyId: 'goat' },
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // When filtering error tests, it should match based on the original vars
+    // not fail because of the extra sessionId property added by GOAT
+    const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-456' });
+
+    // Should find the error test even though sessionId was added
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('goat-test-1');
+  });
+
+  it('should match tests with both _conversation and sessionId runtime vars', async () => {
+    // Combined scenario with both runtime vars
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'combined-test',
+          vars: { input: 'hello' },
+          assert: [],
+        },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-789',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // Both runtime vars added during evaluation
+            vars: {
+              input: 'hello',
+              _conversation: [{ role: 'user', content: 'hi' }],
+              sessionId: 'session-xyz',
+            },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'combined-test',
+              vars: {
+                input: 'hello',
+                _conversation: [{ role: 'user', content: 'hi' }],
+                sessionId: 'session-xyz',
+              },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 1,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    const result = await filterTests(mockTestSuite, { failing: 'eval-789' });
+
+    // Should match even with both runtime vars present
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('combined-test');
+  });
+
+  it('should match tests when stored results have defaultTest.vars merged', async () => {
+    // Simulate a test suite with defaultTest.vars (as it would be loaded from config)
+    // Note: Fresh tests don't have defaultTest.vars merged yet - that happens in prepareTests
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      defaultTest: {
+        vars: {
+          systemPrompt: 'You are a helpful assistant',
+        },
+      },
+      tests: [
+        {
+          description: 'test-with-defaults',
+          vars: { prompt: 'first test' },
+          assert: [],
+        },
+        {
+          description: 'test-that-passed',
+          vars: { prompt: 'second test' },
+          assert: [],
+        },
+      ],
+    };
+
+    // Stored results have defaultTest.vars merged (as done by prepareTests in evaluator)
+    const mockEval = {
+      id: 'eval-default-vars',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // Stored vars have defaultTest.vars merged
+            vars: { systemPrompt: 'You are a helpful assistant', prompt: 'first test' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'test-with-defaults',
+              vars: { systemPrompt: 'You are a helpful assistant', prompt: 'first test' },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // filterTests should merge defaultTest.vars before comparison
+    const result = await filterTests(mockTestSuite, { failing: 'eval-default-vars' });
+
+    // Should find the failing test even though fresh test.vars doesn't have systemPrompt
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('test-with-defaults');
+  });
+
+  it('should match old results that lack defaultTest.vars via fallback', async () => {
+    // This test verifies the fallback matching for old results:
+    // - Config has defaultTest.vars defined
+    // - But stored results were created without defaultTest.vars merged
+    // - The fallback tries matching without defaults, which succeeds
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      defaultTest: {
+        vars: {
+          systemPrompt: 'You are a helpful assistant',
+          context: 'General knowledge',
+        },
+      },
+      tests: [
+        {
+          description: 'emoji-test',
+          vars: { prompt: '😊' + String.fromCodePoint(0xfe00) }, // Emoji encoded
+          assert: [],
+        },
+      ],
+    };
+
+    // Stored results DON'T have defaultTest.vars (old version or bug)
+    const mockEval = {
+      id: 'eval-missing-defaults',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            // Stored vars only have the test-specific vars, NOT the defaults
+            vars: { prompt: '😊' + String.fromCodePoint(0xfe00) },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: {
+              description: 'emoji-test',
+              vars: { prompt: '😊' + String.fromCodePoint(0xfe00) },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // With the fallback, this SHOULD match because:
+    // 1. First try: testCase.vars after merge: { systemPrompt, context, prompt } vs result.vars: { prompt } - NO MATCH
+    // 2. Fallback: testCase.vars without merge: { prompt } vs result.vars: { prompt } - MATCH!
+    const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-missing-defaults' });
+
+    // Should find the test via fallback matching
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('emoji-test');
+  });
+
+  it('should match when defaultTest.vars is empty or undefined', async () => {
+    // When defaultTest has no vars, the merge should not affect matching
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      defaultTest: {
+        // No vars defined, or vars is empty
+      },
+      tests: [
+        {
+          description: 'basic-test',
+          vars: { prompt: 'test prompt' },
+          assert: [],
+        },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-no-defaults',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { prompt: 'test prompt', sessionId: 'runtime-session' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: {
+              description: 'basic-test',
+              vars: { prompt: 'test prompt' },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // Should match because:
+    // - testCase.vars after merge (no defaults): { prompt }
+    // - result.vars after filtering sessionId: { prompt }
+    const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-no-defaults' });
+
+    expect(result).toHaveLength(1);
+    expect(result[0].description).toBe('basic-test');
+  });
+
+  it('should combine failingOnly and errorsOnly filters (union) when both are provided', async () => {
+    // When both --filter-failing-only and --filter-errors-only are used, combine results
+    const mockTestSuite: TestSuite = {
+      prompts: [],
+      providers: [],
+      tests: [
+        {
+          description: 'error-test',
+          vars: { prompt: 'error prompt' },
+          assert: [],
+        },
+        {
+          description: 'failure-test',
+          vars: { prompt: 'failure prompt' },
+          assert: [],
+        },
+        {
+          description: 'passing-test',
+          vars: { prompt: 'passing prompt' },
+          assert: [],
+        },
+      ],
+    };
+
+    const mockEval = {
+      id: 'eval-combined',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { prompt: 'error prompt' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: { description: 'error-test', vars: { prompt: 'error prompt' }, assert: [] },
+          },
+          {
+            vars: { prompt: 'failure prompt' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: {
+              description: 'failure-test',
+              vars: { prompt: 'failure prompt' },
+              assert: [],
+            },
+          },
+          {
+            vars: { prompt: 'passing prompt' },
+            success: true,
+            failureReason: ResultFailureReason.NONE,
+            testCase: {
+              description: 'passing-test',
+              vars: { prompt: 'passing prompt' },
+              assert: [],
+            },
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 1,
+          failures: 1,
+          errors: 1,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+          },
+        },
+      }),
+    };
+
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+    // Using both filters should return union of failingOnly + errors (but not passing)
+    const result = await filterTests(mockTestSuite, {
+      failingOnly: 'eval-combined',
+      errorsOnly: 'eval-combined',
+    });
+
+    // Should find both error-test and failure-test (union), but not passing-test
+    expect(result).toHaveLength(2);
+    const descriptions = result.map((t) => t.description).sort();
+    expect(descriptions).toEqual(['error-test', 'failure-test']);
+  });
+});

test/util/eval/filterPrompts.test.ts

@@ -0,0 +1,162 @@
+import { describe, expect, it } from 'vitest';
+import { filterPrompts } from '../../../src/util/eval/filterPrompts';
+
+import type { Prompt } from '../../../src/types/index';
+
+describe('filterPrompts', () => {
+  const mockPrompts: Prompt[] = [
+    {
+      id: 'prompt-1',
+      raw: 'Hello {{name}}',
+      label: 'Greeting Prompt',
+    },
+    {
+      id: 'prompt-2',
+      raw: 'Goodbye {{name}}',
+      label: 'Farewell Prompt',
+    },
+    {
+      id: 'prompt-3',
+      raw: 'How are you?',
+      label: 'Question Prompt',
+    },
+    {
+      id: 'prompt-4',
+      raw: 'System message',
+      label: 'System:Admin',
+    },
+    {
+      id: 'prompt-5',
+      raw: 'No label here',
+      label: '', // Empty label
+    },
+  ];
+
+  it('should return all prompts if no filter is provided', () => {
+    const result = filterPrompts(mockPrompts);
+    expect(result).toEqual(mockPrompts);
+  });
+
+  it('should filter prompts by exact label match', () => {
+    const result = filterPrompts(mockPrompts, 'Greeting Prompt');
+    expect(result).toHaveLength(1);
+    expect(result[0].label).toBe('Greeting Prompt');
+  });
+
+  it('should filter prompts by partial label match', () => {
+    const result = filterPrompts(mockPrompts, 'Prompt');
+    expect(result).toHaveLength(3);
+    expect(result.map((p) => p.label)).toEqual([
+      'Greeting Prompt',
+      'Farewell Prompt',
+      'Question Prompt',
+    ]);
+  });
+
+  it('should filter prompts by id', () => {
+    const result = filterPrompts(mockPrompts, 'prompt-1');
+    expect(result).toHaveLength(1);
+    expect(result[0].id).toBe('prompt-1');
+  });
+
+  it('should filter prompts by id pattern', () => {
+    const result = filterPrompts(mockPrompts, 'prompt-[12]');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.id)).toEqual(['prompt-1', 'prompt-2']);
+  });
+
+  it('should match on either id or label', () => {
+    const mixedPrompts: Prompt[] = [
+      { id: 'greeting-v1', raw: 'Hello', label: 'Greeting' },
+      { id: 'farewell-v1', raw: 'Goodbye', label: 'Farewell' },
+    ];
+    const result1 = filterPrompts(mixedPrompts, 'v1');
+    expect(result1).toHaveLength(2);
+    const result2 = filterPrompts(mixedPrompts, 'Greeting');
+    expect(result2).toHaveLength(1);
+    expect(result2[0].label).toBe('Greeting');
+  });
+
+  it('should handle regex patterns', () => {
+    const result = filterPrompts(mockPrompts, '(Greeting|Farewell)');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.label)).toEqual(['Greeting Prompt', 'Farewell Prompt']);
+  });
+
+  it('should handle case-sensitive regex', () => {
+    const result = filterPrompts(mockPrompts, 'greeting');
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle case-sensitive matching (default behavior)', () => {
+    // JavaScript RegExp is case-sensitive by default
+    // Users can use [Gg] or similar patterns for case-insensitive matching
+    const result = filterPrompts(mockPrompts, '[Gg]reeting');
+    expect(result).toHaveLength(1);
+    expect(result[0].label).toBe('Greeting Prompt');
+  });
+
+  it('should return empty array if no prompts match filter', () => {
+    const result = filterPrompts(mockPrompts, 'nonexistent');
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle prompts with empty labels', () => {
+    const result = filterPrompts(mockPrompts, 'prompt-');
+    expect(result).toHaveLength(5);
+  });
+
+  it('should handle prompts without label property', () => {
+    const promptsWithoutLabel: Prompt[] = [
+      {
+        id: 'test-1',
+        raw: 'Test',
+        label: 'Valid Label',
+      },
+      {
+        id: 'test-2',
+        raw: 'Test 2',
+        label: undefined as any, // Simulate missing label
+      },
+    ];
+    const result1 = filterPrompts(promptsWithoutLabel, 'Valid');
+    expect(result1).toHaveLength(1);
+    expect(result1[0].label).toBe('Valid Label');
+    const result2 = filterPrompts(promptsWithoutLabel, 'test-');
+    expect(result2).toHaveLength(2);
+  });
+
+  it('should handle special regex characters in labels', () => {
+    const result = filterPrompts(mockPrompts, 'System:Admin');
+    expect(result).toHaveLength(1);
+    expect(result[0].label).toBe('System:Admin');
+  });
+
+  it('should handle complex regex patterns', () => {
+    const result = filterPrompts(mockPrompts, '^(Greeting|Question).*');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.label)).toEqual(['Greeting Prompt', 'Question Prompt']);
+  });
+
+  it('should handle empty prompts array', () => {
+    const result = filterPrompts([], 'anything');
+    expect(result).toEqual([]);
+  });
+
+  it('should throw on invalid regex with helpful error message', () => {
+    expect(() => filterPrompts(mockPrompts, '[invalid')).toThrow(
+      'Invalid regex pattern for --filter-prompts: "[invalid"',
+    );
+  });
+
+  it('should throw on invalid regex with multiple patterns', () => {
+    expect(() => filterPrompts(mockPrompts, '((')).toThrow(
+      'Invalid regex pattern for --filter-prompts',
+    );
+  });
+
+  it('should filter with word boundaries', () => {
+    const result = filterPrompts(mockPrompts, '\\bPrompt\\b');
+    expect(result).toHaveLength(3);
+  });
+});

test/util/eval/filterProviders.test.ts

@@ -0,0 +1,316 @@
+import { describe, expect, it } from 'vitest';
+import {
+  filterProviderConfigs,
+  filterProviders,
+  getPersistedProviderFilterOptions,
+  getProviderFilterRegexError,
+} from '../../../src/util/eval/filterProviders';
+
+import type { ApiProvider, TestSuiteConfig } from '../../../src/types/index';
+import type { ProviderOptions, ProviderOptionsMap } from '../../../src/types/providers';
+
+describe('getPersistedProviderFilterOptions', () => {
+  it('converts a persisted filter into config resolution options', () => {
+    expect(getPersistedProviderFilterOptions('selected-target')).toEqual({
+      filterProviders: 'selected-target',
+    });
+  });
+
+  it.each([undefined, null])('treats a missing persisted filter as no filter: %j', (value) => {
+    expect(getPersistedProviderFilterOptions(value)).toEqual({});
+  });
+
+  it.each(['', 42, {}, ['echo']])('fails closed on a tampered persisted filter: %j', (value) => {
+    expect(() => getPersistedProviderFilterOptions(value)).toThrow(
+      'Stored provider filter is invalid',
+    );
+  });
+});
+
+describe('getProviderFilterRegexError', () => {
+  it('returns undefined for a valid pattern', () => {
+    expect(getProviderFilterRegexError('selected-.*')).toBeUndefined();
+  });
+
+  it('returns the reason for an invalid pattern', () => {
+    expect(getProviderFilterRegexError('[')).toContain('Invalid regular expression');
+  });
+});
+
+describe('filterProviders', () => {
+  const mockProviders: ApiProvider[] = [
+    {
+      id: () => 'openai:gpt-4',
+      label: 'GPT-4',
+      callApi: async () => ({ output: '' }),
+    },
+    {
+      id: () => 'openai:gpt-3.5-turbo',
+      label: 'GPT-3.5',
+      callApi: async () => ({ output: '' }),
+    },
+    {
+      id: () => 'anthropic:claude-2',
+      label: 'Claude',
+      callApi: async () => ({ output: '' }),
+    },
+    {
+      id: () => 'custom:provider',
+      callApi: async () => ({ output: '' }),
+      // No label
+    },
+  ];
+
+  it('should return all providers if no filter is provided', () => {
+    const result = filterProviders(mockProviders);
+    expect(result).toEqual(mockProviders);
+  });
+
+  it('should filter providers by ID', () => {
+    const result = filterProviders(mockProviders, 'openai');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.id())).toEqual(['openai:gpt-4', 'openai:gpt-3.5-turbo']);
+  });
+
+  it('should filter providers by label', () => {
+    const result = filterProviders(mockProviders, 'GPT');
+    expect(result).toHaveLength(2);
+    expect(result.map((p) => p.label)).toEqual(['GPT-4', 'GPT-3.5']);
+  });
+
+  it('should handle providers without labels', () => {
+    const result = filterProviders(mockProviders, 'custom');
+    expect(result).toHaveLength(1);
+    expect(result[0].id()).toBe('custom:provider');
+  });
+
+  it('should handle regex patterns', () => {
+    const result = filterProviders(mockProviders, '(gpt|claude)');
+    expect(result).toHaveLength(3);
+    expect(result.map((p) => p.id())).toEqual([
+      'openai:gpt-4',
+      'openai:gpt-3.5-turbo',
+      'anthropic:claude-2',
+    ]);
+  });
+
+  it('should return empty array if no providers match filter', () => {
+    const result = filterProviders(mockProviders, 'nonexistent');
+    expect(result).toHaveLength(0);
+  });
+
+  it('should handle case sensitivity', () => {
+    const result = filterProviders(mockProviders, 'GPT');
+    expect(result).toHaveLength(2);
+  });
+});
+
+describe('filterProviderConfigs', () => {
+  describe('string providers', () => {
+    it('should return the string if it matches the filter', () => {
+      const result = filterProviderConfigs('openai:gpt-4', 'openai');
+      expect(result).toBe('openai:gpt-4');
+    });
+
+    it('should return empty array if string does not match', () => {
+      const result = filterProviderConfigs('openai:gpt-4', 'anthropic');
+      expect(result).toEqual([]);
+    });
+
+    it('should handle regex patterns for strings', () => {
+      const result = filterProviderConfigs('openai:gpt-4', 'gpt-[0-9]');
+      expect(result).toBe('openai:gpt-4');
+    });
+  });
+
+  describe('function providers', () => {
+    it('should filter function provider by label', () => {
+      const fn = Object.assign(async () => ({ output: '' }), { label: 'MyFunction' });
+      const result = filterProviderConfigs(fn, 'MyFunction');
+      expect(result).toBe(fn);
+    });
+
+    it('should filter function provider without label by default id', () => {
+      const fn = async () => ({ output: '' });
+      const result = filterProviderConfigs(fn, 'custom-function');
+      expect(result).toBe(fn);
+    });
+
+    it('should return empty array if function label does not match', () => {
+      const fn = Object.assign(async () => ({ output: '' }), { label: 'MyFunction' });
+      const result = filterProviderConfigs(fn, 'OtherFunction');
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe('array of providers', () => {
+    it('should return all providers if no filter is provided', () => {
+      const providers = ['openai:gpt-4', 'anthropic:claude-2'];
+      const result = filterProviderConfigs(providers);
+      expect(result).toEqual(providers);
+    });
+
+    it('should filter string providers in array', () => {
+      const providers = ['openai:gpt-4', 'openai:gpt-3.5-turbo', 'anthropic:claude-2'];
+      const result = filterProviderConfigs(providers, 'openai');
+      expect(result).toEqual(['openai:gpt-4', 'openai:gpt-3.5-turbo']);
+    });
+
+    it('should filter ProviderOptions by id', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'GPT-4' },
+        { id: 'anthropic:claude-2', label: 'Claude' },
+      ];
+      const result = filterProviderConfigs(providers, 'openai');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptions[])[0].id).toBe('openai:gpt-4');
+    });
+
+    it('should filter ProviderOptions by label', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'Dev' },
+        { id: 'openai:gpt-4', label: 'Stage' },
+        { id: 'openai:gpt-4', label: 'Prod' },
+      ];
+      const result = filterProviderConfigs(providers, 'Dev');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptions[])[0].label).toBe('Dev');
+    });
+
+    it('should filter ProviderOptionsMap by key (provider id)', () => {
+      const providers: ProviderOptionsMap[] = [
+        { 'openai:gpt-4': { label: 'Dev' } },
+        { 'anthropic:claude-2': { label: 'Stage' } },
+      ];
+      const result = filterProviderConfigs(providers, 'openai');
+      expect(result).toHaveLength(1);
+    });
+
+    it('should filter ProviderOptionsMap by nested label', () => {
+      const providers: ProviderOptionsMap[] = [
+        { 'openai:gpt-4': { label: 'Dev' } },
+        { 'openai:gpt-4': { label: 'Stage' } },
+      ];
+      const result = filterProviderConfigs(providers, 'Stage');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptionsMap[])[0]['openai:gpt-4'].label).toBe('Stage');
+    });
+
+    it('should handle mixed array of provider types', () => {
+      const fn = Object.assign(async () => ({ output: '' }), { label: 'CustomDev' });
+      const providers: TestSuiteConfig['providers'] = [
+        'openai:gpt-4',
+        { id: 'anthropic:claude-2', label: 'ClaudeDev' },
+        { 'google:gemini': { label: 'GeminiDev' } } as ProviderOptionsMap,
+        fn,
+      ];
+      const result = filterProviderConfigs(providers, 'Dev');
+      expect(result).toHaveLength(3);
+    });
+
+    it('should return empty array if no providers match', () => {
+      const providers = ['openai:gpt-4', 'anthropic:claude-2'];
+      const result = filterProviderConfigs(providers, 'nonexistent');
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe('regex patterns', () => {
+    it('should support complex regex patterns', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'Dev-US' },
+        { id: 'openai:gpt-4', label: 'Dev-EU' },
+        { id: 'openai:gpt-4', label: 'Prod-US' },
+      ];
+      const result = filterProviderConfigs(providers, 'Dev-.*');
+      expect(result).toHaveLength(2);
+    });
+
+    it('should match either id or label with regex', () => {
+      const providers: ProviderOptions[] = [
+        { id: 'openai:gpt-4', label: 'Production' },
+        { id: 'anthropic:claude-sonnet', label: 'Dev' },
+      ];
+      const result = filterProviderConfigs(providers, '(gpt|sonnet)');
+      expect(result).toHaveLength(2);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle empty array', () => {
+      const result = filterProviderConfigs([], 'anything');
+      expect(result).toEqual([]);
+    });
+
+    it('should handle provider without label filtering by id only', () => {
+      const providers: ProviderOptions[] = [{ id: 'openai:gpt-4' }];
+      const result = filterProviderConfigs(providers, 'gpt-4');
+      expect(result).toHaveLength(1);
+    });
+
+    it('should handle ProviderOptionsMap with overridden id', () => {
+      const providers: ProviderOptionsMap[] = [
+        { 'file://custom.js': { id: 'my-custom-id', label: 'Dev' } },
+      ];
+      // Should match the overridden id, not the key
+      const result = filterProviderConfigs(providers, 'my-custom-id');
+      expect(result).toHaveLength(1);
+    });
+
+    it('should filter HTTP providers with config by label (bug report case)', () => {
+      const providers: ProviderOptions[] = [
+        {
+          id: 'https',
+          label: 'direct-haiku-4.5',
+          config: {
+            url: 'http://localhost:3333/assistant',
+            method: 'POST',
+            headers: {
+              Accept: '{{acceptHeader}}',
+              'Content-Type': 'application/json',
+              'X-Assistant-Model': 'anthropic:claude-haiku-4-5-20251001',
+              'X-Conversation-ID': '{{conversationId}}',
+              'X-System-ID': '{{systemId}}',
+            },
+            body: {
+              prompt: '{{prompt}}',
+            },
+          },
+        },
+        {
+          id: 'https',
+          label: 'haiku-4.5',
+          config: {
+            url: 'http://localhost:3333/assistant',
+            method: 'POST',
+            headers: {
+              Accept: '{{acceptHeader}}',
+              'Content-Type': 'application/json',
+              'X-Assistant-Model': 'openrouter:anthropic/claude-haiku-4.5',
+              'X-Conversation-ID': '{{conversationId}}',
+              'X-System-ID': '{{systemId}}',
+            },
+            body: {
+              prompt: '{{prompt}}',
+            },
+            transformResponse: 'json',
+          },
+        },
+      ];
+      const result = filterProviderConfigs(providers, 'direct-haiku-4.5');
+      expect(result).toHaveLength(1);
+      expect((result as ProviderOptions[])[0].label).toBe('direct-haiku-4.5');
+    });
+
+    it('should handle provider with null or empty id', () => {
+      const providers: Array<Partial<ProviderOptions>> = [
+        { id: null as any, label: 'Provider1', config: {} },
+        { id: '', label: 'Provider2', config: {} },
+        { id: undefined, label: 'Provider3', config: {} },
+      ];
+      const result = filterProviderConfigs(providers as any, 'Provider2');
+      expect(result).toHaveLength(1);
+      expect((result as Array<Partial<ProviderOptions>>)[0].label).toBe('Provider2');
+    });
+  });
+});

test/util/eval/filterTests.test.ts

@@ -0,0 +1,797 @@
+import path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import Eval from '../../../src/models/eval';
+import { ResultFailureReason } from '../../../src/types/index';
+import { filterTests } from '../../../src/util/eval/filterTests';
+
+import type { TestCase, TestSuite } from '../../../src/types/index';
+
+vi.mock('../../../src/models/eval', () => ({
+  default: {
+    findById: vi.fn(),
+  },
+}));
+
+describe('filterTests', () => {
+  const mockTestSuite: TestSuite = {
+    prompts: [],
+    providers: [],
+    tests: [
+      {
+        description: 'test1',
+        vars: { var1: 'test1' },
+        assert: [],
+        metadata: { type: 'unit' },
+      },
+      {
+        description: 'test2',
+        vars: { var1: 'test2' },
+        assert: [],
+        metadata: { type: 'integration' },
+      },
+      {
+        description: 'test3',
+        vars: { var1: 'test3' },
+        assert: [],
+        metadata: { type: 'unit' },
+      },
+    ],
+  };
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    const mockEval = {
+      id: 'eval-123',
+      createdAt: new Date().getTime(),
+      config: {},
+      results: [],
+      resultsCount: 0,
+      prompts: [],
+      persisted: true,
+      toEvaluateSummary: vi.fn().mockResolvedValue({
+        version: 2,
+        timestamp: new Date().toISOString(),
+        results: [
+          {
+            vars: { var1: 'test1' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: mockTestSuite.tests![0],
+          },
+          {
+            vars: { var1: 'test2' },
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            testCase: mockTestSuite.tests![1],
+          },
+          {
+            vars: { var1: 'test3' },
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            testCase: mockTestSuite.tests![2],
+          },
+        ],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: {
+          successes: 0,
+          failures: 0,
+          errors: 0,
+          tokenUsage: {
+            total: 0,
+            prompt: 0,
+            completion: 0,
+            cached: 0,
+            numRequests: 0,
+            completionDetails: {
+              reasoning: 0,
+              acceptedPrediction: 0,
+              rejectedPrediction: 0,
+            },
+          },
+        },
+      }),
+    };
+    vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+  });
+
+  it('should return all tests if no options provided', async () => {
+    const result = await filterTests(mockTestSuite, {});
+    expect(result).toEqual(mockTestSuite.tests);
+  });
+
+  it('should return empty array if testSuite has no tests', async () => {
+    const result = await filterTests({ prompts: [], providers: [] }, {});
+    expect(result).toEqual([]);
+  });
+
+  describe('metadata filter', () => {
+    it('should filter tests by metadata key-value pair', async () => {
+      const result = await filterTests(mockTestSuite, { metadata: 'type=unit' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test3']);
+    });
+
+    it('should throw error if metadata filter format is invalid', async () => {
+      await expect(filterTests(mockTestSuite, { metadata: 'invalid' })).rejects.toThrow(
+        '--filter-metadata must be specified in key=value format',
+      );
+    });
+
+    it('should exclude tests without metadata', async () => {
+      const testSuite = {
+        ...mockTestSuite,
+        tests: [...mockTestSuite.tests!, { description: 'no-metadata', vars: {}, assert: [] }],
+      };
+      const result = await filterTests(testSuite, { metadata: 'type=unit' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test3']);
+    });
+
+    describe('multiple metadata filters', () => {
+      const multiMetadataTestSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [
+          {
+            description: 'test1',
+            vars: { var1: 'test1' },
+            assert: [],
+            metadata: { type: 'unit', env: 'dev', priority: 'high' },
+          },
+          {
+            description: 'test2',
+            vars: { var1: 'test2' },
+            assert: [],
+            metadata: { type: 'unit', env: 'prod', priority: 'low' },
+          },
+          {
+            description: 'test3',
+            vars: { var1: 'test3' },
+            assert: [],
+            metadata: { type: 'integration', env: 'dev', priority: 'high' },
+          },
+          {
+            description: 'test4',
+            vars: { var1: 'test4' },
+            assert: [],
+            metadata: { type: 'integration', env: 'prod', priority: 'medium' },
+          },
+        ],
+      };
+
+      it('should filter tests matching ALL metadata conditions (AND logic)', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit', 'env=dev'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+
+      it('should return empty when no tests match all conditions', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit', 'env=staging'],
+        });
+        expect(result).toHaveLength(0);
+      });
+
+      it('should handle single string value (backward compatibility)', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: 'type=unit',
+        });
+        expect(result).toHaveLength(2);
+        expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+      });
+
+      it('should handle array with single element', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit'],
+        });
+        expect(result).toHaveLength(2);
+        expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+      });
+
+      it('should filter with three or more conditions', async () => {
+        const result = await filterTests(multiMetadataTestSuite, {
+          metadata: ['type=unit', 'env=dev', 'priority=high'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+
+      it('should handle values containing equals sign', async () => {
+        const testSuiteWithEquals: TestSuite = {
+          prompts: [],
+          providers: [],
+          tests: [
+            {
+              description: 'test-with-equals',
+              vars: { var1: 'test1' },
+              assert: [],
+              metadata: { query: 'a=1&b=2', type: 'special' },
+            },
+          ],
+        };
+        const result = await filterTests(testSuiteWithEquals, {
+          metadata: ['query=a=1&b=2'],
+        });
+        expect(result).toHaveLength(1);
+      });
+
+      it('should throw error if any filter in array is invalid', async () => {
+        await expect(
+          filterTests(multiMetadataTestSuite, {
+            metadata: ['type=unit', 'invalid'],
+          }),
+        ).rejects.toThrow('--filter-metadata must be specified in key=value format');
+      });
+
+      it('should throw error for empty value', async () => {
+        await expect(
+          filterTests(multiMetadataTestSuite, {
+            metadata: ['type='],
+          }),
+        ).rejects.toThrow('--filter-metadata must be specified in key=value format');
+      });
+
+      it('should exclude tests missing any of the required metadata keys', async () => {
+        const testSuitePartialMetadata: TestSuite = {
+          prompts: [],
+          providers: [],
+          tests: [
+            {
+              description: 'has-both',
+              vars: { var1: 'test1' },
+              assert: [],
+              metadata: { type: 'unit', env: 'dev' },
+            },
+            {
+              description: 'missing-env',
+              vars: { var1: 'test2' },
+              assert: [],
+              metadata: { type: 'unit' },
+            },
+          ],
+        };
+        const result = await filterTests(testSuitePartialMetadata, {
+          metadata: ['type=unit', 'env=dev'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+
+      it('should work with array metadata values', async () => {
+        const testSuiteArrayMetadata: TestSuite = {
+          prompts: [],
+          providers: [],
+          tests: [
+            {
+              description: 'has-security-tag',
+              vars: { var1: 'test1' },
+              assert: [],
+              metadata: { type: 'unit', tags: ['security', 'auth'] },
+            },
+            {
+              description: 'no-security-tag',
+              vars: { var1: 'test2' },
+              assert: [],
+              metadata: { type: 'unit', tags: ['performance'] },
+            },
+          ],
+        };
+        const result = await filterTests(testSuiteArrayMetadata, {
+          metadata: ['type=unit', 'tags=security'],
+        });
+        expect(result).toHaveLength(1);
+        expect(result[0]?.vars?.var1).toBe('test1');
+      });
+    });
+  });
+
+  describe('failing filter', () => {
+    it('should filter failing tests when failing option is provided', async () => {
+      // --filter-failing returns all non-successful results (failures + errors)
+      const result = await filterTests(mockTestSuite, { failing: 'eval-123' });
+      expect(result).toHaveLength(3);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2', 'test3']);
+    });
+
+    it('should match failing tests when provider paths differ', async () => {
+      vi.resetAllMocks();
+      const absPath = path.join(process.cwd(), 'provider.js');
+      const mockEval = {
+        id: 'eval-123',
+        createdAt: new Date().getTime(),
+        config: {},
+        results: [],
+        resultsCount: 0,
+        prompts: [],
+        persisted: true,
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          version: 2,
+          timestamp: new Date().toISOString(),
+          results: [
+            {
+              vars: { var1: 'test1' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              provider: { id: `file://${absPath}` },
+              testCase: mockTestSuite.tests![0],
+            },
+          ],
+          table: { head: { prompts: [], vars: [] }, body: [] },
+          stats: {
+            successes: 0,
+            failures: 0,
+            errors: 0,
+            tokenUsage: {
+              total: 0,
+              prompt: 0,
+              completion: 0,
+              cached: 0,
+              numRequests: 0,
+              completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+            },
+          },
+        }),
+      };
+
+      vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+      const testSuite = {
+        ...mockTestSuite,
+        tests: [
+          {
+            ...mockTestSuite.tests![0],
+            provider: `file://./provider.js`,
+          },
+        ],
+      } as TestSuite;
+
+      const result = await filterTests(testSuite, { failing: 'eval-123' });
+      expect(result).toHaveLength(1);
+    });
+  });
+
+  describe('errors only filter', () => {
+    it('should filter error tests when errorsOnly option is provided', async () => {
+      const result = await filterTests(mockTestSuite, { errorsOnly: 'eval-123' });
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test2');
+    });
+  });
+
+  describe('failing only filter', () => {
+    it('should filter assertion failures only, excluding errors', async () => {
+      // --filter-failing-only returns only assertion failures, not errors
+      const result = await filterTests(mockTestSuite, { failingOnly: 'eval-123' });
+      expect(result).toHaveLength(2);
+      // test1 and test3 have ASSERT failures, test2 has ERROR
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test3']);
+    });
+
+    it('should return empty when all failures are errors', async () => {
+      vi.resetAllMocks();
+      const mockEval = {
+        id: 'eval-456',
+        createdAt: new Date().getTime(),
+        config: {},
+        results: [],
+        resultsCount: 0,
+        prompts: [],
+        persisted: true,
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          version: 2,
+          timestamp: new Date().toISOString(),
+          results: [
+            {
+              vars: { var1: 'test1' },
+              success: false,
+              failureReason: ResultFailureReason.ERROR,
+              testCase: mockTestSuite.tests![0],
+            },
+            {
+              vars: { var1: 'test2' },
+              success: false,
+              failureReason: ResultFailureReason.ERROR,
+              testCase: mockTestSuite.tests![1],
+            },
+          ],
+          table: { head: { prompts: [], vars: [] }, body: [] },
+          stats: {
+            successes: 0,
+            failures: 0,
+            errors: 2,
+            tokenUsage: {
+              total: 0,
+              prompt: 0,
+              completion: 0,
+              cached: 0,
+              numRequests: 0,
+              completionDetails: { reasoning: 0, acceptedPrediction: 0, rejectedPrediction: 0 },
+            },
+          },
+        }),
+      };
+      vi.mocked(Eval.findById).mockResolvedValue(mockEval as any);
+
+      const result = await filterTests(mockTestSuite, { failingOnly: 'eval-456' });
+      expect(result).toHaveLength(0);
+    });
+
+    it('should combine failingOnly and errorsOnly when both provided', async () => {
+      // When both are provided, it should be a union of assertion failures and errors
+      const result = await filterTests(mockTestSuite, {
+        failingOnly: 'eval-123',
+        errorsOnly: 'eval-123',
+      });
+      // Should include all 3 tests: test1 (ASSERT), test2 (ERROR), test3 (ASSERT)
+      expect(result).toHaveLength(3);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(
+        expect.arrayContaining(['test1', 'test2', 'test3']),
+      );
+    });
+  });
+
+  describe('pattern filter', () => {
+    it('should filter tests by description pattern', async () => {
+      const result = await filterTests(mockTestSuite, { pattern: 'test[12]' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+    });
+
+    it('should handle tests without description', async () => {
+      const testSuite = {
+        ...mockTestSuite,
+        tests: [...mockTestSuite.tests!, { vars: {}, assert: [] }],
+      };
+      const result = await filterTests(testSuite, { pattern: 'test' });
+      expect(result).toHaveLength(3);
+    });
+
+    it('should throw error for invalid regex pattern', async () => {
+      await expect(filterTests(mockTestSuite, { pattern: '[invalid' })).rejects.toThrow(
+        /Invalid regex pattern "\[invalid"/,
+      );
+    });
+  });
+
+  describe('range filter', () => {
+    it('should slice tests by zero-based start-inclusive, end-exclusive range', async () => {
+      const result = await filterTests(mockTestSuite, { range: '1:3' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test2', 'test3']);
+    });
+
+    it('should support omitted start and omitted end', async () => {
+      const fromStart = await filterTests(mockTestSuite, { range: ':2' });
+      expect(fromStart.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+
+      const toEnd = await filterTests(mockTestSuite, { range: '1:' });
+      expect(toEnd.map((t: TestCase) => t.vars?.var1)).toEqual(['test2', 'test3']);
+    });
+
+    it('should allow range ends beyond the test count', async () => {
+      const result = await filterTests(mockTestSuite, { range: '2:100' });
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test3');
+    });
+
+    it('should apply after pattern and before firstN', async () => {
+      const result = await filterTests(mockTestSuite, {
+        pattern: 'test',
+        range: '1:3',
+        firstN: 1,
+      });
+
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test2');
+    });
+
+    it('should throw error for invalid range format', async () => {
+      await expect(filterTests(mockTestSuite, { range: '1' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+      await expect(filterTests(mockTestSuite, { range: ':' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+      await expect(filterTests(mockTestSuite, { range: 'a:b' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+    });
+
+    it('should reject invalid whitespace-heavy ranges without excessive backtracking', async () => {
+      await expect(
+        filterTests(mockTestSuite, { range: `${'\t'.repeat(10_000)}:${'\t'.repeat(10_000)}x` }),
+      ).rejects.toThrow(/--filter-range must be specified in start:end format/);
+    });
+
+    it('should throw error when start is greater than end', async () => {
+      await expect(filterTests(mockTestSuite, { range: '3:1' })).rejects.toThrow(
+        '--filter-range start must be less than or equal to end, got: 3:1',
+      );
+    });
+
+    it('should reject negative range bounds', async () => {
+      await expect(filterTests(mockTestSuite, { range: '-1:5' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+      await expect(filterTests(mockTestSuite, { range: '0:-3' })).rejects.toThrow(
+        /--filter-range must be specified in start:end format/,
+      );
+    });
+
+    it('should warn when range slices to an empty set on a non-empty suite', async () => {
+      const warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => undefined as any);
+      try {
+        const result = await filterTests(mockTestSuite, { range: '100:200' });
+        expect(result).toHaveLength(0);
+        expect(warnSpy).toHaveBeenCalledWith(
+          expect.stringContaining('--filter-range 100:200 selected 0 tests'),
+        );
+      } finally {
+        warnSpy.mockRestore();
+      }
+    });
+  });
+
+  describe('firstN filter', () => {
+    it('should take first N tests', async () => {
+      const result = await filterTests(mockTestSuite, { firstN: 2 });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+    });
+
+    it('should handle string input for firstN', async () => {
+      const result = await filterTests(mockTestSuite, { firstN: '2' });
+      expect(result).toHaveLength(2);
+      expect(result.map((t: TestCase) => t.vars?.var1)).toEqual(['test1', 'test2']);
+    });
+
+    it('should throw error if firstN is not a number', async () => {
+      await expect(filterTests(mockTestSuite, { firstN: 'invalid' })).rejects.toThrow(
+        'firstN must be a number, got: invalid',
+      );
+    });
+
+    it('should handle undefined firstN', async () => {
+      const result = await filterTests(mockTestSuite, { firstN: undefined });
+      expect(result).toEqual(mockTestSuite.tests);
+    });
+
+    it('should throw error if firstN is null', async () => {
+      await expect(filterTests(mockTestSuite, { firstN: null as any })).rejects.toThrow(
+        'firstN must be a number, got: null',
+      );
+    });
+
+    it('should throw error if firstN is NaN', async () => {
+      await expect(filterTests(mockTestSuite, { firstN: Number.NaN })).rejects.toThrow(
+        'firstN must be a number, got: NaN',
+      );
+    });
+  });
+
+  describe('sample filter', () => {
+    it('should take N random tests', async () => {
+      const result = await filterTests(mockTestSuite, { sample: 2 });
+      expect(result).toHaveLength(2);
+      // Can't test exact values since it's random
+      expect(result.every((t: TestCase) => mockTestSuite.tests!.includes(t))).toBe(true);
+    });
+
+    it('should handle string input for sample', async () => {
+      const result = await filterTests(mockTestSuite, { sample: '2' });
+      expect(result).toHaveLength(2);
+      expect(result.every((t: TestCase) => mockTestSuite.tests!.includes(t))).toBe(true);
+    });
+
+    it('should return the same sample when given the same seed', async () => {
+      const randomSpy = vi.spyOn(Math, 'random').mockImplementation(() => {
+        throw new Error('seeded sampling should not use Math.random');
+      });
+      try {
+        const first = await filterTests(mockTestSuite, { sample: 2, sampleSeed: 42 });
+        const second = await filterTests(mockTestSuite, { sample: 2, sampleSeed: 42 });
+
+        expect(first.map((test) => test.vars?.var1)).toEqual(second.map((test) => test.vars?.var1));
+        expect(randomSpy).not.toHaveBeenCalled();
+      } finally {
+        randomSpy.mockRestore();
+      }
+    });
+
+    it('should throw error if sample is not a number', async () => {
+      await expect(filterTests(mockTestSuite, { sample: 'invalid' })).rejects.toThrow(
+        'sample must be a number, got: invalid',
+      );
+    });
+
+    it('should handle undefined sample', async () => {
+      const result = await filterTests(mockTestSuite, { sample: undefined });
+      expect(result).toEqual(mockTestSuite.tests);
+    });
+
+    it('should throw error if sample is null', async () => {
+      await expect(filterTests(mockTestSuite, { sample: null as any })).rejects.toThrow(
+        'sample must be a number, got: null',
+      );
+    });
+
+    it('should throw error if sample is NaN', async () => {
+      await expect(filterTests(mockTestSuite, { sample: Number.NaN })).rejects.toThrow(
+        'sample must be a number, got: NaN',
+      );
+    });
+  });
+
+  describe('multiple filters', () => {
+    it('should compose metadata and failing filters', async () => {
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: { var1: 'test1' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: mockTestSuite.tests![0],
+            },
+            {
+              vars: { var1: 'test2' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: mockTestSuite.tests![1],
+            },
+            {
+              vars: { var1: 'test3' },
+              success: true,
+              testCase: mockTestSuite.tests![2],
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(mockTestSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result).toHaveLength(1);
+      expect(result[0]?.vars?.var1).toBe('test1');
+    });
+
+    it('should not reintroduce metadata-excluded tests with matching vars', async () => {
+      const excludedTest: TestCase = {
+        description: 'integration test',
+        vars: { prompt: 'shared input' },
+        metadata: { type: 'integration' },
+        provider: 'provider-integration',
+      };
+      const selectedTest: TestCase = {
+        description: 'unit test',
+        vars: { prompt: 'shared input' },
+        metadata: { type: 'unit' },
+        provider: 'provider-unit',
+      };
+      const testSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [excludedTest, selectedTest],
+      };
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: excludedTest.vars,
+              provider: { id: 'provider-integration' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: excludedTest,
+            },
+            {
+              vars: selectedTest.vars,
+              provider: { id: 'provider-unit' },
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: selectedTest,
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(testSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result).toEqual([selectedTest]);
+    });
+
+    it('should preserve generated result tests when metadata matches all config tests', async () => {
+      const configTest: TestCase = {
+        description: 'configured test',
+        vars: { prompt: 'configured' },
+        metadata: { type: 'unit' },
+      };
+      const generatedTest: TestCase = {
+        description: 'generated test',
+        vars: { prompt: 'generated' },
+        metadata: { type: 'unit', pluginId: 'cipher-code' },
+      };
+      const testSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [configTest],
+      };
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: configTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: configTest,
+            },
+            {
+              vars: generatedTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: generatedTest,
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(testSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result.map((test) => test.description)).toEqual(['configured test', 'generated test']);
+    });
+
+    it('should filter generated result tests by metadata before deduplication', async () => {
+      const excludedGeneratedTest: TestCase = {
+        description: 'excluded generated test',
+        vars: { prompt: 'shared generated input' },
+        metadata: { type: 'integration' },
+      };
+      const selectedGeneratedTest: TestCase = {
+        description: 'selected generated test',
+        vars: { prompt: 'shared generated input' },
+        metadata: { type: 'unit' },
+      };
+      const testSuite: TestSuite = {
+        prompts: [],
+        providers: [],
+        tests: [],
+      };
+      vi.mocked(Eval.findById).mockResolvedValue({
+        toEvaluateSummary: vi.fn().mockResolvedValue({
+          results: [
+            {
+              vars: excludedGeneratedTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: excludedGeneratedTest,
+            },
+            {
+              vars: selectedGeneratedTest.vars,
+              success: false,
+              failureReason: ResultFailureReason.ASSERT,
+              testCase: selectedGeneratedTest,
+            },
+          ],
+        }),
+      } as any);
+
+      const result = await filterTests(testSuite, {
+        metadata: 'type=unit',
+        failing: 'eval-123',
+      });
+
+      expect(result).toEqual([selectedGeneratedTest]);
+    });
+  });
+});

test/util/eval/redteamWarning.test.ts

@@ -0,0 +1,63 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../../src/logger';
+import { warnIfRedteamConfigHasNoTests } from '../../../src/util/eval/redteamWarning';
+
+import type { TestSuite, UnifiedConfig } from '../../../src/types/index';
+
+vi.mock('../../../src/logger', () => {
+  const mockLogger = {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  };
+
+  return {
+    __esModule: true,
+    default: mockLogger,
+  };
+});
+
+describe('redteam warning in eval command', () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should warn when config has redteam section but no test cases', () => {
+    const config = {
+      redteam: {
+        purpose: 'Test red team purpose',
+      },
+    } as Partial<UnifiedConfig>;
+    const testSuite = {
+      prompts: [],
+      providers: [],
+      tests: [],
+    } as TestSuite;
+
+    warnIfRedteamConfigHasNoTests(config, testSuite);
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining('redteam section but no test cases'),
+    );
+    expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('promptfoo redteam generate'));
+  });
+
+  it('should not warn when scenarios are present even if tests is empty', () => {
+    const config = {
+      redteam: {
+        purpose: 'Test red team purpose',
+      },
+    } as Partial<UnifiedConfig>;
+    const testSuite = {
+      prompts: [],
+      providers: [],
+      tests: [],
+      scenarios: [{ config: [], tests: [] }],
+    } as TestSuite;
+
+    warnIfRedteamConfigHasNoTests(config, testSuite);
+
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+});

test/util/eval/summary.test.ts

@@ -0,0 +1,891 @@
+import chalk from 'chalk';
+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
+import { generateEvalSummary } from '../../../src/util/eval/summary';
+import { stripAnsi } from '../../util/utils';
+
+import type { EvalSummaryParams } from '../../../src/util/eval/summary';
+import type { TokenUsageTracker } from '../../../src/util/tokenUsage';
+
+type MockTracker = {
+  getProviderIds: Mock;
+  getProviderUsage: Mock;
+  trackUsage: Mock;
+  resetAllUsage: Mock;
+  resetProviderUsage: Mock;
+  getTotalUsage: Mock;
+  cleanup: Mock;
+};
+
+function createMockTracker(): TokenUsageTracker {
+  return {
+    getProviderIds: vi.fn().mockReturnValue([]),
+    getProviderUsage: vi.fn(),
+    trackUsage: vi.fn(),
+    resetAllUsage: vi.fn(),
+    resetProviderUsage: vi.fn(),
+    getTotalUsage: vi.fn(),
+    cleanup: vi.fn(),
+  } as unknown as TokenUsageTracker;
+}
+
+describe('generateEvalSummary', () => {
+  let mockTracker: MockTracker & TokenUsageTracker;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockTracker = createMockTracker() as MockTracker & TokenUsageTracker;
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('completion message', () => {
+    it('should show basic completion message when not writing to database', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-123',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Eval complete');
+      expect(output).not.toContain('eval-123');
+    });
+
+    it('should show eval ID when writing to database without shareable URL', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-456',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Eval complete (ID: eval-456)');
+    });
+
+    it('should show shareable URL when available', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-789',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: 'https://promptfoo.app/eval/abc123',
+        wantsToShare: true,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Eval complete: https://promptfoo.app/eval/abc123');
+      expect(output).not.toContain('eval-789');
+    });
+
+    it('should say "Red team complete" for red team evals', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-rt-1',
+        isRedteam: true,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 10,
+        failures: 2,
+        errors: 0,
+        duration: 8000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✓ Red team complete');
+      expect(output).not.toContain('Eval complete');
+    });
+
+    it('should explain non-retryable target errors when an eval is aborted', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-aborted',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 0,
+        failures: 0,
+        errors: 1,
+        duration: 1000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+        targetErrorStatus: 401,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('✗ Eval aborted (ID: eval-aborted)');
+      expect(output).toContain(
+        'Scan stopped: Target is unavailable and will not recover on retry.',
+      );
+      expect(output).toContain('Target returned HTTP 401');
+      expect(output).not.toContain('Server error (500)');
+      expect(output).toContain('To fix: Check your target configuration and credentials.');
+    });
+  });
+
+  describe('token usage', () => {
+    it('should display eval tokens correctly', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-123',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          prompt: 400,
+          completion: 600,
+          cached: 0,
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion)');
+    });
+
+    it('should display grading tokens only when no eval tokens (critical bug fix)', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-grading-only',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 0,
+          assertions: {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+            cached: 0,
+          },
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Grading: 500 (200 prompt, 300 completion)');
+      expect(output).not.toContain('Eval:');
+    });
+
+    it('should display both eval and grading tokens', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-both',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          prompt: 400,
+          completion: 600,
+          assertions: {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+          },
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion)');
+      expect(output).toContain('Grading: 500 (200 prompt, 300 completion)');
+    });
+
+    it('should show 100% cached correctly', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-cached',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          cached: 1000,
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Tokens:');
+      expect(output).toContain('Eval: 1,000 (cached)');
+    });
+
+    it('should show partial cached tokens', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-partial-cache',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: {
+          total: 1000,
+          prompt: 400,
+          completion: 600,
+          cached: 200,
+        },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion, 200 cached)');
+    });
+
+    it('should not show token section when no tokens', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-no-tokens',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).not.toContain('Tokens:');
+    });
+  });
+
+  describe('provider breakdown', () => {
+    it('should show provider breakdown with request counts', () => {
+      mockTracker.getProviderIds.mockReturnValue(['openai:gpt-4', 'anthropic:claude-3']);
+      mockTracker.getProviderUsage.mockImplementation((id: string) => {
+        if (id === 'openai:gpt-4') {
+          return {
+            total: 1500,
+            prompt: 600,
+            completion: 900,
+            cached: 0,
+            numRequests: 5,
+          };
+        }
+        if (id === 'anthropic:claude-3') {
+          return {
+            total: 800,
+            prompt: 300,
+            completion: 500,
+            cached: 0,
+            numRequests: 3,
+          };
+        }
+        return undefined;
+      });
+
+      const params: EvalSummaryParams = {
+        evalId: 'eval-providers',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 2300 },
+        successes: 8,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Providers:');
+      expect(output).toContain('openai:gpt-4');
+      expect(output).toContain('1,500 (5 requests; 600 prompt, 900 completion)');
+      expect(output).toContain('anthropic:claude-3');
+      expect(output).toContain('800 (3 requests; 300 prompt, 500 completion)');
+    });
+
+    it('should always show request count even when 0', () => {
+      mockTracker.getProviderIds.mockReturnValue(['openai:gpt-4', 'anthropic:claude-3']);
+      mockTracker.getProviderUsage.mockImplementation((id: string) => {
+        if (id === 'openai:gpt-4') {
+          return {
+            total: 1000,
+            cached: 1000,
+            numRequests: 0,
+          };
+        }
+        if (id === 'anthropic:claude-3') {
+          return {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+            numRequests: 2,
+          };
+        }
+        return undefined;
+      });
+
+      const params: EvalSummaryParams = {
+        evalId: 'eval-zero-requests',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 1500 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Providers:');
+      expect(output).toContain('openai:gpt-4: 1,000 (0 requests; cached)');
+      expect(output).toContain('anthropic:claude-3: 500 (2 requests; 200 prompt, 300 completion)');
+    });
+  });
+
+  describe('pass rate and results', () => {
+    it('should show percentages for each result line at 100% pass rate', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-100',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 10,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('Results:');
+      expect(plainOutput).toContain('10 passed (100%)');
+      expect(plainOutput).toContain('0 failed (0%)');
+      expect(plainOutput).toContain('0 errors (0%)');
+    });
+
+    it('should show percentages for each result line when some tests fail', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-85',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 17,
+        failures: 3,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('Results:');
+      expect(plainOutput).toContain('17 passed (85.00%)');
+      expect(plainOutput).toContain('3 failed (15.00%)');
+      expect(plainOutput).toContain('0 errors (0%)');
+    });
+
+    it('should show percentages for each result line when passed and failed are split evenly', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-50',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 5,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('Results:');
+      expect(plainOutput).toContain('5 passed (50.00%)');
+      expect(plainOutput).toContain('5 failed (50.00%)');
+      expect(plainOutput).toContain('0 errors (0%)');
+    });
+
+    it('should include errors in results', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-errors',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 8,
+        failures: 1,
+        errors: 1,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+      const outputLines = plainOutput.split('\n');
+      const hasLineMatching = (pattern: RegExp) => outputLines.some((line) => pattern.test(line));
+
+      expect(plainOutput).toContain('Results:');
+      expect(hasLineMatching(/^\s*(✓\s+)?8 passed \(80\.00%\)$/)).toBe(true);
+      expect(hasLineMatching(/^\s*(✗\s+)?1 failed \(10\.00%\)$/)).toBe(true);
+      expect(hasLineMatching(/^\s*(✗\s+)?1 error \(10\.00%\)$/)).toBe(true);
+      expect(plainOutput).not.toContain('1 errors');
+    });
+
+    it('should render colored icons with muted percentages', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-styling',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 8,
+        failures: 1,
+        errors: 1,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+
+      expect(lines).toContain(
+        `  ${chalk.green('✓')} ${chalk.white.bold('8')} ${chalk.white('passed')} ${chalk.gray('(80.00%)')}`,
+      );
+      expect(lines).toContain(
+        `  ${chalk.red('✗')} ${chalk.white.bold('1')} ${chalk.white('failed')} ${chalk.gray('(10.00%)')}`,
+      );
+      expect(lines).toContain(
+        `  ${chalk.red('✗')} ${chalk.white.bold('1')} ${chalk.white('error')} ${chalk.gray('(10.00%)')}`,
+      );
+    });
+  });
+
+  describe('guidance messages', () => {
+    it('should show guidance when writing to database without shareable URL', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-view',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('» View results: promptfoo view');
+      expect(output).toContain('» Share with your team: https://promptfoo.app');
+      expect(output).toContain('» Feedback: https://promptfoo.dev/feedback');
+    });
+
+    it('should show share guidance with cloud enabled', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-share-cloud',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: true,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('» View results: promptfoo view');
+      expect(output).toContain('» Create shareable URL: promptfoo share');
+      expect(output).not.toContain('https://promptfoo.app');
+    });
+
+    it('should NOT show share guidance when explicitly disabled (--no-share)', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-no-share',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: true,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('» View results: promptfoo view');
+      expect(output).not.toContain('» Share with your team');
+      expect(output).not.toContain('» Create shareable URL');
+    });
+
+    it('should NOT show guidance when not writing to database', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-no-write',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).not.toContain('» View results:');
+      expect(output).not.toContain('» Share');
+      expect(output).not.toContain('» Feedback:');
+    });
+
+    it('should NOT show guidance when shareable URL is present', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-with-url',
+        isRedteam: false,
+        writeToDatabase: true,
+        shareableUrl: 'https://promptfoo.app/eval/abc123',
+        wantsToShare: true,
+        hasExplicitDisable: false,
+        cloudEnabled: true,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).not.toContain('» View results:');
+      expect(output).not.toContain('» Share');
+      expect(output).not.toContain('» Feedback:');
+    });
+  });
+
+  describe('performance metrics', () => {
+    it('should show duration and concurrency', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-perf',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 125, // 125 seconds = 2m 5s
+        maxConcurrency: 8,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      expect(output).toContain('Duration:');
+      expect(output).toContain('(concurrency: 8)');
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should use singular "error" when there is exactly 1 error', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-singular-error',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 1,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('1 error');
+      expect(plainOutput).not.toContain('1 errors');
+    });
+
+    it('should use plural "errors" when there are multiple errors', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-plural-errors',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 3,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('3 errors');
+    });
+
+    it('should use plural "errors" when there are 0 errors', () => {
+      const params: EvalSummaryParams = {
+        evalId: 'eval-zero-errors',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 0 },
+        successes: 5,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      const lines = generateEvalSummary(params);
+      const plainOutput = stripAnsi(lines.join('\n'));
+
+      expect(plainOutput).toContain('0 errors');
+    });
+
+    it('should handle provider returning undefined usage gracefully', () => {
+      mockTracker.getProviderIds.mockReturnValue([
+        'openai:gpt-4',
+        'missing-provider',
+        'anthropic:claude-3',
+      ]);
+      mockTracker.getProviderUsage.mockImplementation((id: string) => {
+        if (id === 'openai:gpt-4') {
+          return {
+            total: 1000,
+            prompt: 400,
+            completion: 600,
+            numRequests: 5,
+          };
+        }
+        if (id === 'missing-provider') {
+          return undefined; // Simulates a provider that returns undefined
+        }
+        if (id === 'anthropic:claude-3') {
+          return {
+            total: 500,
+            prompt: 200,
+            completion: 300,
+            numRequests: 3,
+          };
+        }
+        return undefined;
+      });
+
+      const params: EvalSummaryParams = {
+        evalId: 'eval-undefined-provider',
+        isRedteam: false,
+        writeToDatabase: false,
+        shareableUrl: null,
+        wantsToShare: false,
+        hasExplicitDisable: false,
+        cloudEnabled: false,
+        tokenUsage: { total: 1500 },
+        successes: 8,
+        failures: 0,
+        errors: 0,
+        duration: 5000,
+        maxConcurrency: 4,
+        tracker: mockTracker,
+      };
+
+      // Should not throw
+      const lines = generateEvalSummary(params);
+      const output = stripAnsi(lines.join('\n'));
+
+      // Should show the providers that have valid usage
+      expect(output).toContain('Providers:');
+      expect(output).toContain('openai:gpt-4');
+      expect(output).toContain('anthropic:claude-3');
+      // Should NOT show the missing provider
+      expect(output).not.toContain('missing-provider');
+    });
+  });
+});