chore: import upstream snapshot with attribution

test/factories/data/eval/database_records.ts

@@ -0,0 +1,243 @@
+export function oldStyleEval() {
+  return {
+    id: 'eval-2024-10-01T18:24:51',
+    createdAt: 1727807091574,
+    description: 'Simple test',
+    results: {
+      version: 2,
+      timestamp: '2024-10-01T18:24:51.574Z',
+      results: [
+        {
+          provider: {
+            id: 'openai:gpt-4o-mini',
+            label: '',
+          },
+          prompt: {
+            raw: 'Rephrase this in French: Hello world',
+            label: 'Rephrase this in {{language}}: {{body}}',
+          },
+          vars: {
+            language: 'French',
+            body: 'Hello world',
+          },
+          response: {
+            output: 'Bonjour le monde.',
+            tokenUsage: {
+              total: 19,
+              prompt: 15,
+              completion: 4,
+            },
+            cached: false,
+            cost: 0.0000046499999999999995,
+          },
+          success: true,
+          score: 1,
+          namedScores: {},
+          latencyMs: 684,
+          cost: 0.0000046499999999999995,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'No assertions',
+            tokensUsed: {
+              total: 0,
+              prompt: 0,
+              completion: 0,
+              cached: 0,
+            },
+          },
+        },
+        {
+          provider: {
+            id: 'openai:gpt-4o-mini',
+            label: '',
+          },
+          prompt: {
+            raw: 'Translate this to conversational French: Hello world',
+            label: 'Translate this to conversational {{language}}: {{body}}',
+          },
+          vars: {
+            language: 'French',
+            body: 'Hello world',
+          },
+          response: {
+            output: 'Salut tout le monde !',
+            tokenUsage: {
+              total: 20,
+              prompt: 15,
+              completion: 5,
+            },
+            cached: false,
+            cost: 0.0000052500000000000006,
+          },
+          success: true,
+          score: 1,
+          namedScores: {},
+          latencyMs: 683,
+          cost: 0.0000052500000000000006,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'No assertions',
+            tokensUsed: {
+              total: 0,
+              prompt: 0,
+              completion: 0,
+              cached: 0,
+            },
+          },
+        },
+      ],
+      stats: {
+        successes: 2,
+        failures: 0,
+        tokenUsage: {
+          total: 39,
+          prompt: 30,
+          completion: 9,
+          cached: 0,
+        },
+      },
+      table: {
+        head: {
+          prompts: [
+            {
+              raw: 'Rephrase this in {{language}}: {{body}}',
+              label: 'Rephrase this in {{language}}: {{body}}',
+              id: 'dd3415036994f2d82d0ba70cc5bb2caed82da6ccbdaca8a2174cf3992141c207',
+              provider: 'openai:gpt-4o-mini',
+              metrics: {
+                score: 1,
+                testPassCount: 1,
+                testFailCount: 0,
+                assertPassCount: 0,
+                assertFailCount: 0,
+                totalLatencyMs: 684,
+                tokenUsage: {
+                  total: 19,
+                  prompt: 15,
+                  completion: 4,
+                  cached: 0,
+                },
+                namedScores: {},
+                namedScoresCount: {},
+                cost: 0.0000046499999999999995,
+              },
+            },
+            {
+              raw: 'Translate this to conversational {{language}}: {{body}}',
+              label: 'Translate this to conversational {{language}}: {{body}}',
+              id: '68f67e9473a10ba8e5294074e90b8cc9daa9aed6256e89391e4ad7adbffba52c',
+              provider: 'openai:gpt-4o-mini',
+              metrics: {
+                score: 1,
+                testPassCount: 1,
+                testFailCount: 0,
+                assertPassCount: 0,
+                assertFailCount: 0,
+                totalLatencyMs: 683,
+                tokenUsage: {
+                  total: 20,
+                  prompt: 15,
+                  completion: 5,
+                  cached: 0,
+                },
+                namedScores: {},
+                namedScoresCount: {},
+                cost: 0.0000052500000000000006,
+              },
+            },
+          ],
+          vars: ['body', 'language'],
+        },
+        body: [
+          {
+            outputs: [
+              {
+                pass: true,
+                score: 1,
+                namedScores: {},
+                text: 'Bonjour le monde.',
+                prompt: 'Rephrase this in French: Hello world',
+                provider: 'openai:gpt-4o-mini',
+                latencyMs: 684,
+                tokenUsage: {
+                  total: 19,
+                  prompt: 15,
+                  completion: 4,
+                },
+                gradingResult: {
+                  pass: true,
+                  score: 1,
+                  reason: 'No assertions',
+                  tokensUsed: {
+                    total: 0,
+                    prompt: 0,
+                    completion: 0,
+                    cached: 0,
+                  },
+                },
+                cost: 0.0000046499999999999995,
+              },
+              {
+                pass: true,
+                score: 1,
+                namedScores: {},
+                text: 'Salut tout le monde !',
+                prompt: 'Translate this to conversational French: Hello world',
+                provider: 'openai:gpt-4o-mini',
+                latencyMs: 683,
+                tokenUsage: {
+                  total: 20,
+                  prompt: 15,
+                  completion: 5,
+                },
+                gradingResult: {
+                  pass: true,
+                  score: 1,
+                  reason: 'No assertions',
+                  tokensUsed: {
+                    total: 0,
+                    prompt: 0,
+                    completion: 0,
+                    cached: 0,
+                  },
+                },
+                cost: 0.0000052500000000000006,
+              },
+            ],
+            test: {
+              vars: {
+                language: 'French',
+                body: 'Hello world',
+              },
+              assert: [],
+              options: {},
+              metadata: {},
+            },
+            vars: ['Hello world', 'French'],
+          },
+        ],
+      },
+    },
+    config: {
+      description: 'Simple test',
+      prompts: [
+        'Rephrase this in {{language}}: {{body}}',
+        'Translate this to conversational {{language}}: {{body}}',
+      ],
+      providers: ['openai:gpt-4o-mini'],
+      tests: [
+        {
+          vars: {
+            language: 'French',
+            body: 'Hello world',
+          },
+        },
+      ],
+      sharing: true,
+      extensions: [],
+    },
+    author: 'test@example.com',
+  };
+}

test/factories/data/fakeDataFactory.ts

@@ -0,0 +1,80 @@
+const directories = [
+  '/Applications',
+  '/bin',
+  '/boot',
+  '/boot/defaults',
+  '/dev',
+  '/etc',
+  '/etc/defaults',
+  '/etc/mail',
+  '/etc/namedb',
+  '/etc/periodic',
+  '/etc/ppp',
+  '/home',
+  '/home/user',
+  '/home/user/dir',
+  '/lib',
+  '/Library',
+  '/lost+found',
+  '/media',
+  '/mnt',
+  '/net',
+  '/Network',
+  '/opt',
+  '/opt/bin',
+  '/opt/include',
+  '/opt/lib',
+  '/opt/sbin',
+  '/opt/share',
+  '/private',
+  '/private/tmp',
+  '/private/var',
+  '/proc',
+  '/rescue',
+  '/root',
+  '/sbin',
+  '/selinux',
+  '/srv',
+  '/sys',
+  '/System',
+  '/tmp',
+  '/Users',
+  '/usr',
+  '/usr/X11R6',
+  '/usr/bin',
+  '/usr/include',
+  '/usr/lib',
+  '/usr/libdata',
+  '/usr/libexec',
+  '/usr/local/bin',
+  '/usr/local/src',
+  '/usr/obj',
+  '/usr/ports',
+  '/usr/sbin',
+  '/usr/share',
+  '/usr/src',
+  '/var',
+  '/var/log',
+  '/var/mail',
+  '/var/spool',
+  '/var/tmp',
+  '/var/yp',
+];
+
+class FakeSystemDataFactory {
+  static directoryPath() {
+    // Select a random number of directories
+    const numDirs = Math.floor(Math.random() * 3) + 1;
+    const dirs = directories.slice(0, numDirs);
+    return dirs.join('/');
+  }
+}
+
+/**
+ * A factory for generating fake data modeled on fakerjs.
+ */
+class FakeDataFactory {
+  static system = FakeSystemDataFactory;
+}
+
+export default FakeDataFactory;

test/factories/eval.ts

@@ -0,0 +1,157 @@
+import {
+  type CompletedPrompt,
+  type EvaluateResult,
+  type EvaluateStats,
+  type EvaluateSummaryV2,
+  type EvaluateSummaryV3,
+  type EvaluateTable,
+  type EvaluateTableOutput,
+  type EvaluateTableRow,
+  type PromptMetrics,
+  ResultFailureReason,
+} from '../../src/types/index';
+import { createGradingResult } from './gradingResult';
+import { createProviderResponse, createRequiredTokenUsage, createTokenUsage } from './provider';
+import { createAtomicTestCase, createPrompt } from './testSuite';
+
+export function createPromptMetrics(overrides: Partial<PromptMetrics> = {}): PromptMetrics {
+  return {
+    score: 1,
+    testPassCount: 1,
+    testFailCount: 0,
+    testErrorCount: 0,
+    assertPassCount: 0,
+    assertFailCount: 0,
+    totalLatencyMs: 0,
+    tokenUsage: createTokenUsage(),
+    namedScores: {},
+    namedScoresCount: {},
+    cost: 0,
+    ...overrides,
+  };
+}
+
+export function createCompletedPrompt(
+  raw = 'Test prompt',
+  overrides: Partial<CompletedPrompt> = {},
+): CompletedPrompt {
+  // raw/label are resolved by createPrompt — pull them out of the override
+  // spread so the remaining fields layer on top without clobbering the
+  // derived label.
+  const { raw: _rawOverride, label: _labelOverride, ...rest } = overrides;
+  return {
+    ...createPrompt(raw, overrides),
+    provider: 'test-provider',
+    metrics: createPromptMetrics(),
+    ...rest,
+  };
+}
+
+export function createEvaluateResult(overrides: Partial<EvaluateResult> = {}): EvaluateResult {
+  // prompt and promptId are derived together — promptId defaults to the
+  // prompt's raw value unless the caller supplies one. Pull both out of the
+  // override spread so the remaining fields layer on top without having to
+  // re-pin them afterward.
+  const { prompt: promptOverride, promptId: promptIdOverride, ...rest } = overrides;
+  const prompt = promptOverride ?? createPrompt();
+  return {
+    promptIdx: 0,
+    testIdx: 0,
+    testCase: createAtomicTestCase(),
+    provider: { id: 'test-provider' },
+    vars: {},
+    response: createProviderResponse(),
+    failureReason: ResultFailureReason.NONE,
+    success: true,
+    score: 1,
+    latencyMs: 0,
+    gradingResult: createGradingResult(),
+    namedScores: {},
+    cost: 0,
+    tokenUsage: createRequiredTokenUsage(),
+    ...rest,
+    prompt,
+    promptId: promptIdOverride ?? prompt.raw,
+  };
+}
+
+export function createEvaluateStats(overrides: Partial<EvaluateStats> = {}): EvaluateStats {
+  return {
+    successes: 1,
+    failures: 0,
+    errors: 0,
+    tokenUsage: createRequiredTokenUsage(),
+    ...overrides,
+  };
+}
+
+export function createEvaluateTableOutput(
+  overrides: Partial<EvaluateTableOutput> = {},
+): EvaluateTableOutput {
+  return {
+    cost: 0,
+    failureReason: ResultFailureReason.NONE,
+    gradingResult: createGradingResult(),
+    id: 'test-output',
+    latencyMs: 0,
+    namedScores: {},
+    pass: true,
+    prompt: 'Test prompt',
+    provider: 'test-provider',
+    response: createProviderResponse(),
+    score: 1,
+    testCase: createAtomicTestCase(),
+    text: 'Test output',
+    tokenUsage: createTokenUsage(),
+    ...overrides,
+  };
+}
+
+export function createEvaluateTableRow(
+  overrides: Partial<EvaluateTableRow> = {},
+): EvaluateTableRow {
+  return {
+    outputs: [createEvaluateTableOutput()],
+    vars: [],
+    test: createAtomicTestCase(),
+    testIdx: 0,
+    ...overrides,
+  };
+}
+
+export function createEvaluateTable(overrides: Partial<EvaluateTable> = {}): EvaluateTable {
+  return {
+    head: {
+      prompts: [createCompletedPrompt()],
+      vars: [],
+    },
+    body: [createEvaluateTableRow()],
+    ...overrides,
+  };
+}
+
+export function createEvaluateSummaryV3(
+  overrides: Partial<EvaluateSummaryV3> = {},
+): EvaluateSummaryV3 {
+  return {
+    version: 3,
+    timestamp: new Date(0).toISOString(),
+    results: [createEvaluateResult()],
+    prompts: [createCompletedPrompt()],
+    stats: createEvaluateStats(),
+    ...overrides,
+  };
+}
+
+export function createEvaluateSummaryV2(
+  overrides: Partial<EvaluateSummaryV2> = {},
+): EvaluateSummaryV2 {
+  return {
+    version: 2,
+    timestamp: new Date(0).toISOString(),
+    results: [createEvaluateResult()],
+    table: createEvaluateTable(),
+    stats: createEvaluateStats(),
+    ...overrides,
+  };
+}

test/factories/evalFactory.ts

@@ -0,0 +1,286 @@
+import { randomUUID } from 'crypto';
+
+import { getDb } from '../../src/database/index';
+import { evalsTable } from '../../src/database/tables';
+import { ResultFailureReason } from '../../src/index';
+import Eval from '../../src/models/eval';
+import { oldStyleEval } from './data/eval/database_records';
+
+import type { EvaluateResult } from '../../src/types/index';
+
+interface CreateEvalOptions {
+  numResults?: number;
+  resultTypes?: Array<'success' | 'error' | 'failure'>;
+  withHighlights?: boolean;
+  withNamedScores?: boolean;
+  searchableContent?: string;
+}
+
+export default class EvalFactory {
+  static async create(options?: CreateEvalOptions) {
+    const eval_ = await Eval.create(
+      {
+        providers: [{ id: 'test-provider' }],
+        prompts: ['What is the capital of {{state}}?'],
+        tests: [
+          { vars: { state: 'colorado' }, assert: [{ type: 'contains', value: 'Denver' }] },
+          { vars: { state: 'california' }, assert: [{ type: 'contains', value: 'Sacramento' }] },
+        ],
+      },
+      [
+        { raw: 'What is the capital of california?', label: 'What is the capital of {{state}}?' },
+        { raw: 'What is the capital of colorado?', label: 'What is the capital of {{state}}?' },
+      ],
+      { id: randomUUID() },
+    );
+    await eval_.addPrompts([
+      {
+        raw: 'What is the capital of california?',
+        label: 'What is the capital of {{state}}?',
+        provider: 'test-provider',
+        metrics: {
+          score: 1,
+          testPassCount: 1,
+          testFailCount: 1,
+          testErrorCount: 0,
+          assertPassCount: 1,
+          assertFailCount: 1,
+          totalLatencyMs: 200,
+          tokenUsage: { total: 20, prompt: 10, completion: 10, cached: 0 },
+          namedScores: {},
+          namedScoresCount: {},
+          redteam: {
+            pluginPassCount: {},
+            pluginFailCount: {},
+            strategyPassCount: {},
+            strategyFailCount: {},
+          },
+          cost: 0.007,
+        },
+      },
+    ]);
+
+    // If no options are provided, create the default test results
+    if (!options || options.numResults === undefined) {
+      await this.addDefaultResults(eval_);
+      return eval_;
+    }
+
+    // Generate the specified number of test results with the requested characteristics
+    const numResults = options.numResults === 0 ? 0 : options.numResults || 2;
+    const resultTypes = options.resultTypes || ['success'];
+    const withHighlights = options.withHighlights || false;
+    const withNamedScores = options.withNamedScores || false;
+    const searchableContent = options.searchableContent || '';
+
+    for (let i = 0; i < numResults; i++) {
+      // Cycle through the result types
+      const resultTypeIndex = i % resultTypes.length;
+      const resultType = resultTypes[resultTypeIndex];
+
+      // Create a test result based on the type
+      await this.addCustomResult(eval_, {
+        testIdx: i,
+        resultType,
+        withHighlight: withHighlights && i % 3 === 0, // Add highlights to every 3rd result if enabled
+        withNamedScores,
+        searchableContent,
+      });
+    }
+
+    return eval_;
+  }
+
+  private static async addDefaultResults(eval_: Eval) {
+    await eval_.addResult({
+      description: 'test-description',
+      promptIdx: 0,
+      testIdx: 0,
+      testCase: { vars: { state: 'colorado' }, assert: [{ type: 'contains', value: 'Denver' }] },
+      promptId: 'test-prompt',
+      provider: { id: 'test-provider', label: 'test-label' },
+      prompt: {
+        raw: 'What is the capital of {{state}}?',
+        label: 'What is the capital of {{state}}?',
+      },
+      vars: { state: 'colorado' },
+      response: {
+        output: 'denver',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      },
+      error: null,
+      failureReason: ResultFailureReason.NONE,
+      success: true,
+      score: 1,
+      latencyMs: 100,
+      gradingResult: {
+        pass: true,
+        score: 1,
+        reason: 'Expected output "denver" to equal "Denver"',
+        namedScores: {},
+        tokensUsed: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        componentResults: [
+          {
+            pass: true,
+            score: 1,
+            reason: 'Expected output "denver" to equal "Denver"',
+            assertion: {
+              type: 'equals',
+              value: 'denver',
+            },
+          },
+        ],
+      },
+      namedScores: {},
+      cost: 0.007,
+      metadata: {},
+    });
+
+    await eval_.addResult({
+      description: 'test-description',
+      promptIdx: 0,
+      testIdx: 1,
+      testCase: {
+        vars: { state: 'california' },
+        assert: [{ type: 'contains', value: 'Sacramento' }],
+      },
+      promptId: 'test-prompt',
+      provider: { id: 'test-provider', label: 'test-label' },
+      prompt: {
+        raw: 'What is the capital of {{state}}?',
+        label: 'What is the capital of {{state}}?',
+      },
+      vars: { state: 'california' },
+      response: {
+        output: 'san francisco',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      },
+      error: null,
+      failureReason: ResultFailureReason.NONE,
+      success: false,
+      score: 0,
+      latencyMs: 100,
+      gradingResult: {
+        pass: false,
+        score: 0,
+        reason: 'Expected output "san francisco" to equal "Sacramento"',
+        namedScores: {},
+        tokensUsed: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        componentResults: [
+          {
+            pass: false,
+            score: 0,
+            reason: 'Expected output "san francisco" to equal "Sacramento"',
+            assertion: {
+              type: 'equals',
+              value: 'denver',
+            },
+          },
+        ],
+      },
+      namedScores: {},
+      cost: 0.007,
+      metadata: {},
+    });
+  }
+
+  private static async addCustomResult(
+    eval_: Eval,
+    options: {
+      testIdx: number;
+      resultType: 'success' | 'error' | 'failure';
+      withHighlight?: boolean;
+      withNamedScores?: boolean;
+      searchableContent?: string;
+    },
+  ) {
+    const { testIdx, resultType, withHighlight, withNamedScores, searchableContent } = options;
+    const stateName = `state${testIdx}`;
+    const output =
+      resultType === 'success'
+        ? `Capital of ${stateName} is TestCity${testIdx}`
+        : `Incorrect answer for ${stateName}${searchableContent ? ` ${searchableContent}` : ''}`;
+
+    const result: Partial<EvaluateResult> = {
+      description: `test-${resultType}-${testIdx}`,
+      promptIdx: 0,
+      testIdx,
+      testCase: {
+        vars: { state: stateName },
+        assert: [{ type: 'contains', value: `TestCity${testIdx}` }],
+      },
+      promptId: 'test-prompt',
+      provider: { id: 'test-provider', label: 'test-label' },
+      prompt: {
+        raw: `What is the capital of ${stateName}?`,
+        label: 'What is the capital of {{state}}?',
+      },
+      vars: { state: stateName },
+      response: {
+        output,
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      },
+      success: resultType === 'success',
+      score: resultType === 'success' ? 1 : 0,
+      latencyMs: 100,
+      namedScores: withNamedScores
+        ? { accuracy: resultType === 'success' ? 1 : 0.2, relevance: 0.8 }
+        : {},
+      cost: 0.007,
+      metadata: {},
+    };
+
+    // Handle error type
+    if (resultType === 'error') {
+      result.error = `Error processing ${stateName}: Test error message`;
+      result.failureReason = ResultFailureReason.ERROR;
+    } else {
+      result.error = null;
+      result.failureReason =
+        resultType === 'success' ? ResultFailureReason.NONE : ResultFailureReason.ASSERT;
+    }
+
+    // Configure the grading result
+    if (resultType !== 'error') {
+      let comment = '';
+      if (withHighlight) {
+        comment = `!highlight Important observation for ${stateName}`;
+      }
+
+      result.gradingResult = {
+        pass: resultType === 'success',
+        score: resultType === 'success' ? 1 : 0,
+        reason:
+          resultType === 'success'
+            ? `Output matches expected value for ${stateName}`
+            : `Output doesn't match expected value for ${stateName}`,
+        comment,
+        namedScores: withNamedScores
+          ? { accuracy: resultType === 'success' ? 1 : 0.2, relevance: 0.8 }
+          : {},
+        tokensUsed: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        componentResults: [
+          {
+            pass: resultType === 'success',
+            score: resultType === 'success' ? 1 : 0,
+            reason:
+              resultType === 'success'
+                ? `Expected output matches for ${stateName}`
+                : `Expected output doesn't match for ${stateName}`,
+            assertion: {
+              type: 'equals',
+              value: `TestCity${testIdx}`,
+            },
+          },
+        ],
+      };
+    }
+
+    await eval_.addResult(result as EvaluateResult);
+  }
+
+  static async createOldResult() {
+    const db = await getDb();
+    return (await db.insert(evalsTable).values(oldStyleEval()).returning())[0];
+  }
+}

test/factories/gradingResult.ts

@@ -0,0 +1,51 @@
+import { createProviderResponse } from './provider';
+
+import type { GradingResult, ProviderResponse } from '../../src/types/index';
+
+export function createGradingResult(overrides: Partial<GradingResult> = {}): GradingResult {
+  return {
+    pass: true,
+    score: 1,
+    reason: 'Test grading output',
+    ...overrides,
+  };
+}
+
+export type PassingGradingResult = Readonly<GradingResult> & {
+  readonly pass: true;
+  readonly score: 1;
+};
+
+export type FailingGradingResult = Readonly<GradingResult> & {
+  readonly pass: false;
+  readonly score: 0;
+};
+
+// pass/score are locked after overrides so callers cannot accidentally flip
+// a passing fixture to a failure (or vice versa). reason remains overridable.
+export function createPassingGradingResult(
+  overrides: Partial<GradingResult> = {},
+): PassingGradingResult {
+  return createGradingResult({ ...overrides, pass: true, score: 1 }) as PassingGradingResult;
+}
+
+export function createFailingGradingResult(
+  overrides: Partial<GradingResult> = {},
+): FailingGradingResult {
+  return createGradingResult({
+    reason: 'Grading failed reason',
+    ...overrides,
+    pass: false,
+    score: 0,
+  }) as FailingGradingResult;
+}
+
+export function createGradingProviderResponse(
+  gradingResult: GradingResult = createPassingGradingResult(),
+  overrides: Partial<ProviderResponse> = {},
+): ProviderResponse {
+  return createProviderResponse({
+    output: JSON.stringify(gradingResult),
+    ...overrides,
+  });
+}

test/factories/provider.ts

@@ -0,0 +1,142 @@
+import { vi } from 'vitest';
+
+import type { ApiProvider, ProviderResponse, TokenUsage } from '../../src/types/index';
+
+export type MockApiProvider = ApiProvider & {
+  id: ReturnType<typeof vi.fn<() => string>>;
+  callApi: ReturnType<typeof vi.fn<ApiProvider['callApi']>>;
+  cleanup?: ReturnType<typeof vi.fn<NonNullable<ApiProvider['cleanup']>>>;
+};
+
+export type MockProviderOptions = Omit<Partial<ApiProvider>, 'callApi' | 'cleanup' | 'id'> & {
+  callApi?: ApiProvider['callApi'];
+  cleanup?: ApiProvider['cleanup'] | boolean;
+  id?: string | (() => string);
+  response?: ProviderResponse;
+};
+
+/**
+ * Shallow-replace the default scaffolding with the caller's overrides. Unlike
+ * createRequiredTokenUsage below, this helper does NOT deep-merge nested fields
+ * because TokenUsage leaves completionDetails/assertions optional — if a caller
+ * passes either, they are declaring the whole shape they want to assert against.
+ * Tests in test/redteam/providers/iterativeTree.test.ts rely on this behavior
+ * to check exact tokenUsage propagation, so preserve it.
+ */
+export function createTokenUsage(overrides: TokenUsage = {}): TokenUsage {
+  return {
+    total: 10,
+    prompt: 5,
+    completion: 5,
+    cached: 0,
+    numRequests: 1,
+    ...overrides,
+  };
+}
+
+/**
+ * Returns a Required<TokenUsage> with every nested field populated. Deep-merges
+ * completionDetails and assertions because Required<> forces every field to be
+ * present — partial overrides for those nested bags would otherwise drop
+ * defaults the caller didn't mention. This is the right behavior for fixtures
+ * flowing into type-strict slots (e.g. EvaluateResult.tokenUsage).
+ */
+export function createRequiredTokenUsage(overrides: TokenUsage = {}): Required<TokenUsage> {
+  return {
+    total: 10,
+    prompt: 5,
+    completion: 5,
+    cached: 0,
+    numRequests: 1,
+    ...overrides,
+    completionDetails: {
+      reasoning: 0,
+      acceptedPrediction: 0,
+      rejectedPrediction: 0,
+      cacheReadInputTokens: 0,
+      cacheCreationInputTokens: 0,
+      ...overrides.completionDetails,
+    },
+    assertions: {
+      total: 0,
+      prompt: 0,
+      completion: 0,
+      cached: 0,
+      numRequests: 0,
+      ...overrides.assertions,
+      completionDetails: {
+        reasoning: 0,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+        ...overrides.assertions?.completionDetails,
+      },
+    },
+  };
+}
+
+/**
+ * Returns a default ProviderResponse. tokenUsage follows the shallow-replace
+ * convention from createTokenUsage: callers passing { tokenUsage: {...} } are
+ * declaring the exact shape they want to assert on, so we do not deep-merge
+ * the default scaffolding into it. If you need defaults preserved, wrap the
+ * partial in createTokenUsage(...) at the call site.
+ */
+export function createProviderResponse(
+  overrides: Partial<ProviderResponse> = {},
+): ProviderResponse {
+  return {
+    output: 'Test output',
+    tokenUsage: createTokenUsage(),
+    ...overrides,
+  };
+}
+
+function resolveIdImpl(id: string | (() => string) = 'test-provider'): () => string {
+  return typeof id === 'function' ? id : () => id;
+}
+
+export function createMockProvider(options: MockProviderOptions = {}): MockApiProvider {
+  const {
+    callApi,
+    cleanup,
+    id,
+    response = createProviderResponse(),
+    ...providerOverrides
+  } = options;
+
+  const provider: MockApiProvider = {
+    id: vi.fn<() => string>().mockImplementation(resolveIdImpl(id)),
+    callApi: vi.fn<ApiProvider['callApi']>().mockImplementation(callApi ?? (async () => response)),
+    ...providerOverrides,
+  };
+
+  if (cleanup) {
+    provider.cleanup = vi
+      .fn<NonNullable<ApiProvider['cleanup']>>()
+      .mockImplementation(cleanup === true ? async () => undefined : cleanup);
+  }
+
+  return provider;
+}
+
+export type ResetMockProviderOptions = {
+  id?: string | (() => string);
+  response?: ProviderResponse;
+  cleanup?: ApiProvider['cleanup'] | boolean;
+};
+
+export function resetMockProvider(
+  provider: MockApiProvider,
+  options: ResetMockProviderOptions = {},
+): void {
+  const { id, response = createProviderResponse(), cleanup } = options;
+  provider.id.mockReset().mockImplementation(resolveIdImpl(id));
+  provider.callApi.mockReset().mockResolvedValue(response);
+  if (provider.cleanup) {
+    provider.cleanup
+      .mockReset()
+      .mockImplementation(!cleanup || cleanup === true ? async () => undefined : cleanup);
+  }
+}

test/factories/testFactories.test.ts

@@ -0,0 +1,200 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createCompletedPrompt, createEvaluateResult } from './eval';
+import { createFailingGradingResult, createPassingGradingResult } from './gradingResult';
+import {
+  createMockProvider,
+  createProviderResponse,
+  createRequiredTokenUsage,
+  resetMockProvider,
+} from './provider';
+import { createPrompt } from './testSuite';
+
+describe('test factories', () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('uses the effective raw value as the default prompt label', () => {
+    expect(createPrompt('base prompt', { raw: 'overridden prompt' })).toMatchObject({
+      raw: 'overridden prompt',
+      label: 'overridden prompt',
+    });
+
+    expect(
+      createPrompt('base prompt', { raw: 'overridden prompt', label: 'custom label' }),
+    ).toMatchObject({
+      raw: 'overridden prompt',
+      label: 'custom label',
+    });
+  });
+
+  it('keeps completed prompt labels aligned with overridden raw values', () => {
+    expect(createCompletedPrompt('base prompt', { raw: 'overridden prompt' })).toMatchObject({
+      raw: 'overridden prompt',
+      label: 'overridden prompt',
+    });
+
+    expect(
+      createCompletedPrompt('base prompt', { raw: 'overridden prompt', label: 'custom label' }),
+    ).toMatchObject({
+      raw: 'overridden prompt',
+      label: 'custom label',
+    });
+  });
+
+  it('uses the effective prompt raw value as the default evaluate result prompt id', () => {
+    expect(createEvaluateResult({ prompt: createPrompt('overridden prompt') })).toMatchObject({
+      prompt: {
+        raw: 'overridden prompt',
+        label: 'overridden prompt',
+      },
+      promptId: 'overridden prompt',
+    });
+
+    expect(
+      createEvaluateResult({
+        prompt: createPrompt('overridden prompt'),
+        promptId: 'custom prompt id',
+      }),
+    ).toMatchObject({
+      promptId: 'custom prompt id',
+    });
+  });
+
+  it('keeps grading helper pass and score invariants', () => {
+    expect(createPassingGradingResult({ pass: false, score: 0, reason: 'custom' })).toMatchObject({
+      pass: true,
+      score: 1,
+      reason: 'custom',
+    });
+
+    expect(createFailingGradingResult({ pass: true, score: 1, reason: 'custom' })).toMatchObject({
+      pass: false,
+      score: 0,
+      reason: 'custom',
+    });
+  });
+
+  it('preserves required nested token usage defaults when overriding assertion details', () => {
+    expect(
+      createRequiredTokenUsage({
+        assertions: {
+          completionDetails: {
+            reasoning: 7,
+          },
+        },
+      }).assertions.completionDetails,
+    ).toEqual({
+      reasoning: 7,
+      acceptedPrediction: 0,
+      rejectedPrediction: 0,
+      cacheReadInputTokens: 0,
+      cacheCreationInputTokens: 0,
+    });
+  });
+
+  // Regression guard for the ...overrides.assertions merge order in
+  // createRequiredTokenUsage. An earlier revision spread overrides.assertions
+  // *after* the literal completionDetails key, which silently clobbered the
+  // fully-populated nested defaults. The spread must happen before the
+  // literal so the object-literal key wins — this test locks that ordering.
+  it('merges scalar and nested assertion overrides without clobbering completionDetails', () => {
+    const result = createRequiredTokenUsage({
+      assertions: {
+        total: 5,
+        prompt: 3,
+        completionDetails: { reasoning: 7 },
+      },
+    });
+
+    expect(result.assertions).toEqual({
+      total: 5,
+      prompt: 3,
+      completion: 0,
+      cached: 0,
+      numRequests: 0,
+      completionDetails: {
+        reasoning: 7,
+        acceptedPrediction: 0,
+        rejectedPrediction: 0,
+        cacheReadInputTokens: 0,
+        cacheCreationInputTokens: 0,
+      },
+    });
+  });
+
+  describe('createMockProvider', () => {
+    it('resolves the default callApi to createProviderResponse', async () => {
+      const provider = createMockProvider();
+      expect(provider.id()).toBe('test-provider');
+      await expect(provider.callApi('prompt')).resolves.toEqual(createProviderResponse());
+    });
+
+    it('accepts a function id that resolves lazily', () => {
+      let counter = 0;
+      const provider = createMockProvider({ id: () => `provider-${++counter}` });
+      expect(provider.id()).toBe('provider-1');
+      expect(provider.id()).toBe('provider-2');
+    });
+
+    it('invokes a user-supplied callApi implementation instead of the default', async () => {
+      const customCallApi = vi.fn(async () => createProviderResponse({ output: 'custom' }));
+      const provider = createMockProvider({ callApi: customCallApi });
+      await expect(provider.callApi('prompt')).resolves.toMatchObject({ output: 'custom' });
+      expect(customCallApi).toHaveBeenCalledWith('prompt');
+    });
+
+    it('wires cleanup as a resolved no-op when cleanup is true', async () => {
+      const provider = createMockProvider({ cleanup: true });
+      expect(provider.cleanup).toBeDefined();
+      await expect(provider.cleanup!()).resolves.toBeUndefined();
+    });
+
+    it('wires cleanup to a user-supplied implementation when cleanup is a function', async () => {
+      const cleanupImpl = vi.fn(async () => undefined);
+      const provider = createMockProvider({ cleanup: cleanupImpl });
+      await provider.cleanup!();
+      expect(cleanupImpl).toHaveBeenCalledTimes(1);
+    });
+
+    it('does not attach cleanup when the option is omitted', () => {
+      const provider = createMockProvider();
+      expect(provider.cleanup).toBeUndefined();
+    });
+  });
+
+  describe('resetMockProvider', () => {
+    it('re-installs the default id and callApi resolution after reset', async () => {
+      const provider = createMockProvider({
+        id: 'original',
+        callApi: async () => createProviderResponse({ output: 'original' }),
+      });
+      await provider.callApi('prompt');
+      expect(provider.callApi).toHaveBeenCalledTimes(1);
+
+      resetMockProvider(provider);
+
+      expect(provider.callApi).toHaveBeenCalledTimes(0);
+      expect(provider.id()).toBe('test-provider');
+      await expect(provider.callApi('prompt')).resolves.toEqual(createProviderResponse());
+    });
+
+    it('re-installs an explicit response and id override', async () => {
+      const provider = createMockProvider();
+      resetMockProvider(provider, {
+        id: 'reset-provider',
+        response: createProviderResponse({ output: 'reset output' }),
+      });
+      expect(provider.id()).toBe('reset-provider');
+      await expect(provider.callApi('prompt')).resolves.toMatchObject({ output: 'reset output' });
+    });
+
+    it('re-seeds cleanup when the shared mock was constructed with cleanup: true', async () => {
+      const provider = createMockProvider({ cleanup: true });
+      const replacement = vi.fn(async () => undefined);
+      resetMockProvider(provider, { cleanup: replacement });
+      await provider.cleanup!();
+      expect(replacement).toHaveBeenCalledTimes(1);
+    });
+  });
+});

test/factories/testSuite.ts

@@ -0,0 +1,35 @@
+import { createMockProvider } from './provider';
+
+import type { AtomicTestCase, Prompt, TestCase, TestSuite } from '../../src/types/index';
+
+export function createPrompt(raw = 'Test prompt', overrides: Partial<Prompt> = {}): Prompt {
+  const effectiveRaw = overrides.raw ?? raw;
+  return {
+    ...overrides,
+    raw: effectiveRaw,
+    label: overrides.label ?? effectiveRaw,
+  };
+}
+
+export function createTestCase(overrides: Partial<TestCase> = {}): TestCase {
+  return {
+    vars: {},
+    ...overrides,
+  };
+}
+
+export function createAtomicTestCase(overrides: Partial<AtomicTestCase> = {}): AtomicTestCase {
+  return {
+    vars: {},
+    ...overrides,
+  };
+}
+
+export function createTestSuite(overrides: Partial<TestSuite> = {}): TestSuite {
+  return {
+    providers: [createMockProvider()],
+    prompts: [createPrompt()],
+    tests: [createTestCase()],
+    ...overrides,
+  };
+}