chore: import upstream snapshot with attribution

test/node/evaluationStore.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import Eval from '../../src/models/eval';
+import EvalResult from '../../src/models/evalResult';
+import { EvalEvaluationStore } from '../../src/node/evaluationStore';
+import { createCompletedPrompt, createEvaluateResult } from '../factories/eval';
+
+describe('EvalEvaluationStore', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('exposes the underlying evaluation state', () => {
+    const evaluation = new Eval({ description: 'adapter test' });
+    const store = new EvalEvaluationStore(evaluation);
+
+    expect(store.evaluation).toBe(evaluation);
+    expect(store.id).toBe(evaluation.id);
+    expect(store.config).toBe(evaluation.config);
+    expect(store.persisted).toBe(false);
+    expect(store.prompts).toBe(evaluation.prompts);
+    expect(store.results).toBe(evaluation.results);
+    expect(store.resultPersistenceFailed).toBe(false);
+  });
+
+  it('delegates result, prompt, resume, and read operations', async () => {
+    const evaluation = new Eval({});
+    const store = new EvalEvaluationStore(evaluation);
+    const result = createEvaluateResult();
+    const prompts = [createCompletedPrompt('Prompt')];
+    const modelResult = { testIdx: 0 } as EvalResult;
+    const completed = new Set(['0:0']);
+
+    const addResult = vi.spyOn(evaluation, 'addResult').mockResolvedValue(undefined);
+    const addPrompts = vi.spyOn(evaluation, 'addPrompts').mockResolvedValue(undefined);
+    const getCompletedIndexPairs = vi
+      .spyOn(EvalResult, 'getCompletedIndexPairs')
+      .mockResolvedValue(completed);
+    const getFailedResultsByTestIdx = vi
+      .spyOn(evaluation, 'getFailedResultsByTestIdx')
+      .mockResolvedValue([modelResult]);
+    const getResults = vi.spyOn(evaluation, 'getResults').mockResolvedValue([modelResult]);
+    const fetchResultsByTestIdx = vi
+      .spyOn(evaluation, 'fetchResultsByTestIdx')
+      .mockResolvedValue([modelResult]);
+
+    await store.appendResult(result);
+    await store.appendPrompts(prompts);
+
+    expect(await store.readCompletedIndexPairs({ excludeErrors: true })).toBe(completed);
+    expect(await store.readFailedResultsByTestIdx(0)).toEqual([modelResult]);
+    expect(await store.readResults()).toEqual([modelResult]);
+    expect(await store.readResultsByTestIdx(0)).toEqual([modelResult]);
+    expect(addResult).toHaveBeenCalledWith(result);
+    expect(addPrompts).toHaveBeenCalledWith(prompts);
+    expect(getCompletedIndexPairs).toHaveBeenCalledWith(evaluation.id, { excludeErrors: true });
+    expect(getFailedResultsByTestIdx).toHaveBeenCalledWith(0);
+    expect(getResults).toHaveBeenCalledOnce();
+    expect(fetchResultsByTestIdx).toHaveBeenCalledWith(0);
+  });
+
+  it('delegates persistence failure, final result, and save operations', async () => {
+    const evaluation = new Eval({});
+    const store = new EvalEvaluationStore(evaluation);
+    const result = createEvaluateResult();
+    const modelResult = { save: vi.fn().mockResolvedValue(undefined) } as unknown as EvalResult;
+
+    const recordFinalJsonlResult = vi.spyOn(evaluation, 'recordFinalJsonlResult');
+    const recordResultPersistenceFailure = vi.spyOn(evaluation, 'recordResultPersistenceFailure');
+    const hasResultPersistenceFailure = vi
+      .spyOn(evaluation, 'hasResultPersistenceFailure')
+      .mockReturnValue(true);
+    const save = vi.spyOn(evaluation, 'save').mockResolvedValue(undefined);
+    const setDurationMs = vi.spyOn(evaluation, 'setDurationMs');
+    const setVars = vi.spyOn(evaluation, 'setVars');
+
+    store.recordFinalResult(result);
+    store.recordResultPersistenceFailure(result);
+    expect(store.hasResultPersistenceFailure(result)).toBe(true);
+    await store.saveResult(modelResult);
+    store.setDurationMs(100);
+    store.setVars(['topic']);
+    await store.save();
+
+    expect(recordFinalJsonlResult).toHaveBeenCalledWith(result);
+    expect(recordResultPersistenceFailure).toHaveBeenCalledWith(result);
+    expect(hasResultPersistenceFailure).toHaveBeenCalledWith(result);
+    expect(modelResult.save).toHaveBeenCalledOnce();
+    expect(setDurationMs).toHaveBeenCalledWith(100);
+    expect(setVars).toHaveBeenCalledWith(['topic']);
+    expect(save).toHaveBeenCalledOnce();
+  });
+
+  it('normalizes model results while preserving plain EvaluateResult values', () => {
+    const store = new EvalEvaluationStore(new Eval({}));
+    const plainResult = createEvaluateResult();
+    const normalizedResult = createEvaluateResult({ score: 0.5 });
+    const modelResult = {
+      toEvaluateResult: vi.fn().mockReturnValue(normalizedResult),
+    } as unknown as EvalResult;
+
+    expect(store.toEvaluateResult(plainResult)).toBe(plainResult);
+    expect(store.toEvaluateResult(modelResult)).toBe(normalizedResult);
+    expect(modelResult.toEvaluateResult).toHaveBeenCalledOnce();
+  });
+});

test/node/evaluatorRuntime.test.ts

@@ -0,0 +1,73 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import Eval from '../../src/models/eval';
+import { nodeEvaluatorRuntime } from '../../src/node/evaluatorRuntime';
+
+import type { EvaluateResult } from '../../src/types/index';
+
+describe('nodeEvaluatorRuntime', () => {
+  const tempDirs: string[] = [];
+
+  afterEach(() => {
+    vi.clearAllMocks();
+    for (const tempDir of tempDirs.splice(0)) {
+      // On Windows, file handles can linger briefly after a stream is closed, so an
+      // immediate recursive delete may throw ENOTEMPTY/EBUSY/EPERM. Retry with a short
+      // backoff (a no-op on POSIX, where these errors don't occur).
+      fs.rmSync(tempDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
+    }
+  });
+
+  it('creates and closes JSONL writers for JSONL output paths only', async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-evaluator-runtime-'));
+    tempDirs.push(tempDir);
+    const jsonlPath = path.join(tempDir, 'results.jsonl');
+    const uppercaseJsonlPath = path.join(tempDir, 'uppercase.JSONL');
+    const csvPath = path.join(tempDir, 'results.csv');
+
+    const writers = nodeEvaluatorRuntime.createResultWriters(
+      [jsonlPath, uppercaseJsonlPath, csvPath],
+      { append: false },
+    );
+
+    expect(writers).toHaveLength(2);
+    await writers[0].write({ output: 'hello' });
+    await writers[1].write({ output: 'uppercase' });
+    await writers[0].close();
+    await writers[1].close();
+    expect(fs.readFileSync(jsonlPath, 'utf8')).toBe('{"output":"hello"}\n');
+    expect(fs.readFileSync(uppercaseJsonlPath, 'utf8')).toBe('{"output":"uppercase"}\n');
+  });
+
+  it('truncates by default and appends when resuming', async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-evaluator-runtime-'));
+    tempDirs.push(tempDir);
+    const jsonlPath = path.join(tempDir, 'results.jsonl');
+
+    fs.writeFileSync(jsonlPath, 'stale row\n');
+    const [writer] = nodeEvaluatorRuntime.createResultWriters(jsonlPath, { append: false });
+    await writer.write({ output: 'fresh' });
+    await writer.close();
+    expect(fs.readFileSync(jsonlPath, 'utf8')).toBe('{"output":"fresh"}\n');
+
+    const [appender] = nodeEvaluatorRuntime.createResultWriters(jsonlPath, { append: true });
+    await appender.write({ output: 'resumed' });
+    await appender.close();
+    expect(fs.readFileSync(jsonlPath, 'utf8')).toBe('{"output":"fresh"}\n{"output":"resumed"}\n');
+  });
+
+  it('creates an Eval-backed evaluation store', async () => {
+    const result = { success: true } as EvaluateResult;
+    const evaluation = new Eval({});
+    const addResult = vi.spyOn(evaluation, 'addResult').mockResolvedValue(undefined);
+    const store = nodeEvaluatorRuntime.createEvaluationStore(evaluation);
+
+    await store.appendResult(result);
+
+    expect(store.evaluation).toBe(evaluation);
+    expect(addResult).toHaveBeenCalledWith(result);
+  });
+});

test/node/retry.test.ts

@@ -0,0 +1,558 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import cliState from '../../src/cliState';
+import { evaluate } from '../../src/evaluator';
+import logger from '../../src/logger';
+import Eval from '../../src/models/eval';
+import { notifyEvaluationChanged } from '../../src/models/evalMutation';
+import {
+  deleteErrorResults,
+  getErrorResultIds,
+  recalculatePromptMetrics,
+  retryCommand,
+} from '../../src/node/retry';
+import { createShareableUrl, isSharingEnabled } from '../../src/share';
+import { ResultFailureReason } from '../../src/types/index';
+import { resolveConfigs } from '../../src/util/config/load';
+import { writeMultipleOutputs } from '../../src/util/output';
+import { shouldShareResults } from '../../src/util/sharing';
+
+import type { TestSuite, UnifiedConfig } from '../../src/types/index';
+
+const dbMocks = vi.hoisted(() => {
+  const errorRows: Array<{ id: string }> = [];
+  const affectedEvalRows: Array<{ evalId: string }> = [];
+  const errorRowsAll = vi.fn(async () => errorRows);
+  const affectedEvalRowsAll = vi.fn(async () => affectedEvalRows);
+  const deleteRun = vi.fn(async () => undefined);
+  const db = {
+    select: vi.fn(() => ({
+      from: vi.fn(() => ({
+        where: vi.fn(() => ({
+          all: errorRowsAll,
+        })),
+      })),
+    })),
+    selectDistinct: vi.fn(() => ({
+      from: vi.fn(() => ({
+        where: vi.fn(() => ({
+          all: affectedEvalRowsAll,
+        })),
+      })),
+    })),
+    delete: vi.fn(() => ({
+      where: vi.fn(() => ({
+        run: deleteRun,
+      })),
+    })),
+  };
+
+  return {
+    affectedEvalRows,
+    db,
+    deleteRun,
+    errorRows,
+  };
+});
+
+vi.mock('../../src/database/index', () => ({
+  getDb: vi.fn(async () => dbMocks.db),
+}));
+vi.mock('../../src/evaluator');
+vi.mock('../../src/logger');
+vi.mock('../../src/models/eval');
+vi.mock('../../src/models/evalMutation');
+vi.mock('../../src/share');
+vi.mock('../../src/util/config/load', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('../../src/util/config/load')>()),
+  resolveConfigs: vi.fn(),
+}));
+vi.mock('../../src/util/output');
+vi.mock('../../src/util/sharing');
+
+const testSuite = {
+  prompts: [],
+  providers: [
+    {
+      id: () => 'echo',
+      label: 'selected-target',
+      callApi: vi.fn(),
+    },
+  ],
+  tests: [],
+} as unknown as TestSuite;
+
+function createEval(overrides: Partial<Eval> = {}): Eval {
+  return {
+    id: 'eval-123',
+    config: {},
+    persisted: false,
+    prompts: [],
+    addPrompts: vi.fn().mockResolvedValue(undefined),
+    fetchResultsBatched: vi.fn(async function* () {}),
+    ...overrides,
+  } as unknown as Eval;
+}
+
+function mockResolvedConfig({
+  commandLineOptions,
+  config,
+  providers,
+}: {
+  commandLineOptions?: Record<string, unknown>;
+  config?: Record<string, unknown>;
+  providers?: TestSuite['providers'];
+} = {}) {
+  vi.mocked(resolveConfigs).mockResolvedValue({
+    basePath: '/workspace',
+    commandLineOptions,
+    config: (config ?? {}) as UnifiedConfig,
+    testSuite: providers ? { ...testSuite, providers } : testSuite,
+  });
+}
+
+describe('retryCommand', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    dbMocks.errorRows.splice(0);
+    dbMocks.affectedEvalRows.splice(0);
+    cliState.resume = false;
+    cliState.retryMode = false;
+    cliState.maxConcurrency = undefined;
+    vi.mocked(shouldShareResults).mockReturnValue(false);
+    vi.mocked(isSharingEnabled).mockReturnValue(false);
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+    dbMocks.errorRows.splice(0);
+    dbMocks.affectedEvalRows.splice(0);
+    cliState.resume = false;
+    cliState.retryMode = false;
+    cliState.maxConcurrency = undefined;
+  });
+
+  it('rejects a retry for an evaluation that does not exist', async () => {
+    vi.mocked(Eval.findById).mockResolvedValue(undefined);
+
+    await expect(retryCommand('missing-eval', {})).rejects.toThrow(
+      'Evaluation with ID missing-eval not found',
+    );
+
+    expect(resolveConfigs).not.toHaveBeenCalled();
+    expect(evaluate).not.toHaveBeenCalled();
+  });
+
+  it('returns the original evaluation when there are no error results', async () => {
+    const originalEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+
+    await expect(retryCommand(originalEval.id, {})).resolves.toBe(originalEval);
+
+    expect(resolveConfigs).not.toHaveBeenCalled();
+    expect(evaluate).not.toHaveBeenCalled();
+  });
+
+  it('reads and deletes error result ids while notifying affected evaluations', async () => {
+    dbMocks.errorRows.push({ id: 'error-result-1' }, { id: 'error-result-2' });
+    dbMocks.affectedEvalRows.push({ evalId: 'eval-123' }, { evalId: 'eval-456' });
+
+    await expect(getErrorResultIds('eval-123')).resolves.toEqual([
+      'error-result-1',
+      'error-result-2',
+    ]);
+    await deleteErrorResults(['error-result-1', 'error-result-2']);
+
+    expect(dbMocks.deleteRun).toHaveBeenCalledTimes(1);
+    expect(notifyEvaluationChanged).toHaveBeenCalledWith('eval-123');
+    expect(notifyEvaluationChanged).toHaveBeenCalledWith('eval-456');
+  });
+
+  it('skips database work when there are no error result ids to delete', async () => {
+    await deleteErrorResults([]);
+
+    expect(dbMocks.db.selectDistinct).not.toHaveBeenCalled();
+    expect(dbMocks.db.delete).not.toHaveBeenCalled();
+  });
+
+  it('recalculates prompt metrics from batched results and persists them', async () => {
+    const prompts = [{}, {}] as any[];
+    const evalRecord = createEval({
+      persisted: true,
+      prompts,
+      fetchResultsBatched: vi.fn(async function* () {
+        yield [
+          {
+            id: 'pass-result',
+            promptIdx: 0,
+            success: true,
+            failureReason: ResultFailureReason.NONE,
+            score: 1,
+            latencyMs: 20,
+            cost: 0.25,
+            namedScores: { quality: 0.8 },
+            testCase: { vars: { topic: 'security' } },
+            response: {
+              tokenUsage: { total: 10, prompt: 6, completion: 4 },
+            },
+            gradingResult: {
+              pass: true,
+              score: 1,
+              reason: 'passed',
+              componentResults: [{ pass: true }, { pass: false }],
+              tokensUsed: { total: 3, prompt: 2, completion: 1 },
+            },
+          },
+          {
+            id: 'error-result',
+            promptIdx: 0,
+            success: false,
+            failureReason: ResultFailureReason.ERROR,
+            score: 0,
+            latencyMs: 0,
+            namedScores: {},
+          },
+          {
+            id: 'failed-result',
+            promptIdx: 1,
+            success: false,
+            failureReason: ResultFailureReason.ASSERT,
+            score: 0.5,
+            latencyMs: 5,
+            cost: 0.1,
+            namedScores: {},
+          },
+          {
+            id: 'invalid-prompt-result',
+            promptIdx: 99,
+            success: true,
+            failureReason: ResultFailureReason.NONE,
+            score: 1,
+            namedScores: {},
+          },
+        ] as any[];
+      }),
+    });
+
+    await recalculatePromptMetrics(evalRecord);
+
+    expect(prompts[0].metrics).toMatchObject({
+      score: 1,
+      testPassCount: 1,
+      testErrorCount: 1,
+      testFailCount: 0,
+      assertPassCount: 1,
+      assertFailCount: 1,
+      totalLatencyMs: 20,
+      cost: 0.25,
+    });
+    expect(prompts[0].metrics.tokenUsage).toMatchObject({
+      total: 10,
+      prompt: 6,
+      completion: 4,
+      assertions: { total: 3, prompt: 2, completion: 1 },
+    });
+    expect(prompts[1].metrics).toMatchObject({
+      score: 0.5,
+      testPassCount: 0,
+      testErrorCount: 0,
+      testFailCount: 1,
+      totalLatencyMs: 5,
+      cost: 0.1,
+    });
+    expect(evalRecord.addPrompts).toHaveBeenCalledWith(prompts);
+    expect(logger.debug).toHaveBeenCalledWith(
+      'Skipping result with invalid promptIdx: 99',
+      expect.objectContaining({ resultId: 'invalid-prompt-result' }),
+    );
+  });
+
+  it('logs and rethrows metric recalculation and persistence failures', async () => {
+    const calculationError = new Error('batch unavailable');
+    const calculationEval = createEval({
+      prompts: [{}] as any[],
+      fetchResultsBatched: vi.fn(async function* () {
+        throw calculationError;
+      }),
+    });
+
+    await expect(recalculatePromptMetrics(calculationEval)).rejects.toThrow('batch unavailable');
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error during batched metrics recalculation',
+      expect.objectContaining({ error: calculationError }),
+    );
+
+    const persistenceError = new Error('prompt save unavailable');
+    const persistenceEval = createEval({
+      persisted: true,
+      prompts: [{}] as any[],
+      addPrompts: vi.fn().mockRejectedValue(persistenceError),
+    });
+
+    await expect(recalculatePromptMetrics(persistenceEval)).rejects.toThrow(
+      'prompt save unavailable',
+    );
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error saving recalculated prompt metrics',
+      expect.objectContaining({ error: persistenceError }),
+    );
+  });
+
+  it('retries from the saved config, cleans up old errors, and shares the result', async () => {
+    const originalEval = createEval({
+      config: { sharing: false } as UnifiedConfig,
+      runtimeOptions: { providerFilter: 'selected-target' },
+    });
+    const retriedEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    dbMocks.affectedEvalRows.push({ evalId: originalEval.id });
+    mockResolvedConfig({
+      commandLineOptions: { delay: 0, maxConcurrency: 4, share: true },
+      config: { sharing: true },
+    });
+    vi.mocked(evaluate).mockImplementation(async (receivedSuite, receivedEval, options) => {
+      expect(cliState.resume).toBe(true);
+      expect(cliState.retryMode).toBe(true);
+      expect(cliState.maxConcurrency).toBe(4);
+      expect(receivedSuite).toBe(testSuite);
+      expect(receivedEval).toBe(originalEval);
+      expect(options).toEqual({
+        delay: 0,
+        eventSource: 'cli',
+        maxConcurrency: 4,
+        showProgressBar: true,
+      });
+      return retriedEval;
+    });
+    vi.mocked(shouldShareResults).mockReturnValue(true);
+    vi.mocked(isSharingEnabled).mockReturnValue(true);
+    vi.mocked(createShareableUrl).mockResolvedValue('https://example.com/eval/eval-123');
+
+    await expect(retryCommand(originalEval.id, {})).resolves.toBe(retriedEval);
+
+    expect(resolveConfigs).toHaveBeenCalledWith(
+      { filterProviders: 'selected-target' },
+      originalEval.config,
+    );
+    expect(dbMocks.deleteRun).toHaveBeenCalledTimes(1);
+    expect(notifyEvaluationChanged).toHaveBeenCalledWith(originalEval.id);
+    expect(shouldShareResults).toHaveBeenCalledWith({
+      cliShare: undefined,
+      configShare: true,
+      configSharing: true,
+    });
+    expect(createShareableUrl).toHaveBeenCalledWith(retriedEval, { silent: false });
+    expect(cliState.resume).toBe(false);
+    expect(cliState.retryMode).toBe(false);
+    expect(cliState.maxConcurrency).toBeUndefined();
+  });
+
+  it('uses an explicit config and forces concurrency to one when delay is requested', async () => {
+    const originalEval = createEval({
+      runtimeOptions: { providerFilter: 'selected-target' },
+    });
+    const retriedEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig({
+      commandLineOptions: { delay: 5, maxConcurrency: 2 },
+    });
+    vi.mocked(evaluate).mockImplementation(async (_suite, _eval, options) => {
+      expect(cliState.maxConcurrency).toBe(1);
+      expect(options).toEqual({
+        delay: 25,
+        eventSource: 'cli',
+        maxConcurrency: 1,
+        showProgressBar: false,
+      });
+      return retriedEval;
+    });
+
+    await expect(
+      retryCommand(originalEval.id, {
+        config: 'retry.yaml',
+        delay: 25,
+        maxConcurrency: 8,
+        verbose: true,
+      }),
+    ).resolves.toBe(retriedEval);
+
+    expect(resolveConfigs).toHaveBeenCalledWith(
+      { config: ['retry.yaml'], filterProviders: 'selected-target' },
+      {},
+    );
+    expect(logger.info).toHaveBeenCalledWith(
+      'Running at concurrency=1 because 25ms delay was requested between API calls',
+    );
+  });
+
+  it('preserves error results when an explicit config no longer matches the stored filter', async () => {
+    const originalEval = createEval({
+      runtimeOptions: { providerFilter: 'selected-target' },
+    });
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig({ providers: [] });
+
+    await expect(retryCommand(originalEval.id, { config: 'retry.yaml' })).rejects.toThrow(
+      'Stored provider filter "selected-target" matched no providers in the retry config "retry.yaml"',
+    );
+
+    expect(evaluate).not.toHaveBeenCalled();
+    expect(dbMocks.deleteRun).not.toHaveBeenCalled();
+  });
+
+  it('preserves error results when the stored filter cannot be applied to the config', async () => {
+    const originalEval = createEval({
+      runtimeOptions: { providerFilter: '[' },
+    });
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+
+    await expect(retryCommand(originalEval.id, { config: 'retry.yaml' })).rejects.toThrow(
+      'Could not resolve the retry config "retry.yaml" using stored provider filter "[": Invalid regular expression',
+    );
+
+    // The pattern is validated before any config resolution happens.
+    expect(resolveConfigs).not.toHaveBeenCalled();
+    expect(evaluate).not.toHaveBeenCalled();
+    expect(dbMocks.deleteRun).not.toHaveBeenCalled();
+    expect(cliState.resume).toBe(false);
+    expect(cliState.retryMode).toBe(false);
+  });
+
+  it('fails closed when the persisted provider filter is not a string', async () => {
+    const originalEval = createEval({
+      runtimeOptions: { providerFilter: ['selected-target'] as unknown as string },
+    });
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+
+    await expect(retryCommand(originalEval.id, {})).rejects.toThrow(
+      'Stored provider filter is invalid',
+    );
+
+    expect(resolveConfigs).not.toHaveBeenCalled();
+    expect(evaluate).not.toHaveBeenCalled();
+    expect(dbMocks.deleteRun).not.toHaveBeenCalled();
+  });
+
+  it('preserves error results and clears retry state when evaluation fails', async () => {
+    const originalEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig();
+    vi.mocked(evaluate).mockRejectedValue(new Error('provider unavailable'));
+
+    await expect(retryCommand(originalEval.id, {})).rejects.toThrow('provider unavailable');
+
+    expect(dbMocks.deleteRun).not.toHaveBeenCalled();
+    expect(cliState.resume).toBe(false);
+    expect(cliState.retryMode).toBe(false);
+    expect(cliState.maxConcurrency).toBeUndefined();
+  });
+
+  it('restores JSONL output and preserves error rows when retry persistence fails', async () => {
+    const originalEval = createEval({
+      config: { outputPath: ['results.jsonl', 'results.json'] } as UnifiedConfig,
+    });
+    const retriedEval = createEval({ resultPersistenceFailed: true });
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig();
+    vi.mocked(evaluate).mockResolvedValue(retriedEval);
+
+    await expect(retryCommand(originalEval.id, {})).rejects.toThrow(
+      'Retry results failed to persist. Existing ERROR rows were preserved.',
+    );
+
+    expect(writeMultipleOutputs).toHaveBeenCalledWith(['results.jsonl'], retriedEval, null);
+    expect(retriedEval.resultPersistenceFailed).toBe(true);
+    expect(dbMocks.deleteRun).not.toHaveBeenCalled();
+  });
+
+  it('warns when JSONL restoration and post-retry rewriting fail', async () => {
+    const originalEval = createEval({
+      config: { outputPath: 'results.jsonl' } as UnifiedConfig,
+    });
+    const failedPersistenceEval = createEval({ resultPersistenceFailed: true });
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig();
+    vi.mocked(evaluate).mockResolvedValueOnce(failedPersistenceEval);
+    vi.mocked(writeMultipleOutputs).mockRejectedValueOnce(new Error('restore unavailable'));
+
+    await expect(retryCommand(originalEval.id, {})).rejects.toThrow(
+      'Retry results failed to persist. Existing ERROR rows were preserved.',
+    );
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Retry results failed to persist, and restoring JSONL output failed.',
+      expect.objectContaining({ jsonlOutputPaths: ['results.jsonl'] }),
+    );
+
+    const successfulEval = createEval();
+    vi.mocked(evaluate).mockResolvedValueOnce(successfulEval);
+    vi.mocked(writeMultipleOutputs).mockRejectedValueOnce(new Error('rewrite unavailable'));
+
+    await expect(retryCommand(originalEval.id, {})).resolves.toBe(successfulEval);
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Retry succeeded and the database is up to date, but rewriting JSONL output failed.',
+      expect.objectContaining({ outputPaths: ['results.jsonl'] }),
+    );
+  });
+
+  it('keeps a successful retry when post-retry cleanup fails', async () => {
+    const originalEval = createEval();
+    const retriedEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig();
+    vi.mocked(evaluate).mockResolvedValue(retriedEval);
+    dbMocks.deleteRun.mockRejectedValueOnce(new Error('database unavailable'));
+
+    await expect(retryCommand(originalEval.id, {})).resolves.toBe(retriedEval);
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Post-retry cleanup had issues. Retry results are saved.',
+      {
+        error: expect.any(Error),
+      },
+    );
+  });
+
+  it('keeps a successful retry when cloud sync throws', async () => {
+    const originalEval = createEval();
+    const retriedEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig();
+    vi.mocked(evaluate).mockResolvedValue(retriedEval);
+    vi.mocked(shouldShareResults).mockReturnValue(true);
+    vi.mocked(isSharingEnabled).mockReturnValue(true);
+    vi.mocked(createShareableUrl).mockRejectedValue(new Error('cloud unavailable'));
+
+    await expect(retryCommand(originalEval.id, { share: true })).resolves.toBe(retriedEval);
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Cloud sync failed. Run promptfoo share eval-123 to retry manually.',
+    );
+  });
+
+  it('warns when cloud sync returns no URL', async () => {
+    const originalEval = createEval();
+    const retriedEval = createEval();
+    vi.mocked(Eval.findById).mockResolvedValue(originalEval);
+    dbMocks.errorRows.push({ id: 'error-result-1' });
+    mockResolvedConfig();
+    vi.mocked(evaluate).mockResolvedValue(retriedEval);
+    vi.mocked(shouldShareResults).mockReturnValue(true);
+    vi.mocked(isSharingEnabled).mockReturnValue(true);
+    vi.mocked(createShareableUrl).mockResolvedValue(null);
+
+    await expect(retryCommand(originalEval.id, { share: true })).resolves.toBe(retriedEval);
+
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Cloud sync failed. Run promptfoo share eval-123 to retry manually.',
+    );
+  });
+});

test/node/testProvider.test.ts

@@ -0,0 +1,505 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { createMockProvider as createFactoryProvider } from '../factories/provider';
+
+import type { ApiProvider } from '../../src/types/providers';
+
+// Use vi.hoisted to create mock functions that persist across clearAllMocks
+const mockEvaluate = vi.hoisted(() => vi.fn());
+const mockToEvaluateSummary = vi.hoisted(() =>
+  vi.fn().mockResolvedValue({
+    results: [
+      {
+        response: {
+          output: 'Hello! How can I help you?',
+          raw: 'Hello! How can I help you?',
+          sessionId: 'session-123',
+          metadata: { transformedRequest: { body: '{}' } },
+        },
+        error: undefined,
+      },
+    ],
+  }),
+);
+const mockNeverGenerateRemote = vi.hoisted(() => vi.fn().mockReturnValue(false));
+const mockFetchWithProxy = vi.hoisted(() =>
+  vi.fn().mockResolvedValue({
+    ok: true,
+    json: vi.fn().mockResolvedValue({
+      message: 'Test completed successfully',
+      error: null,
+      changes_needed: false,
+    }),
+  }),
+);
+const mockDoRemoteGrading = vi.hoisted(() =>
+  vi.fn().mockResolvedValue({
+    pass: true,
+    reason: 'Session is working correctly',
+  }),
+);
+const mockDetermineEffectiveSessionSource = vi.hoisted(() => vi.fn().mockReturnValue('client'));
+
+vi.mock('dedent', () => ({
+  default: (strings: TemplateStringsArray, ...values: unknown[]) => {
+    let result = '';
+    strings.forEach((str, i) => {
+      result += str + (values[i] ?? '');
+    });
+    return result;
+  },
+}));
+
+vi.mock('../../src/evaluator', () => ({
+  evaluate: mockEvaluate,
+}));
+
+vi.mock('../../src/globalConfig/cloud', () => ({
+  cloudConfig: {
+    getApiHost: vi.fn().mockReturnValue('https://api.example.com'),
+    getApiKey: vi.fn().mockReturnValue('test-api-key'),
+  },
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+vi.mock('../../src/models/eval', () => {
+  return {
+    default: class MockEval {
+      toEvaluateSummary = mockToEvaluateSummary;
+    },
+  };
+});
+
+vi.mock('../../src/redteam/remoteGeneration', () => ({
+  neverGenerateRemote: mockNeverGenerateRemote,
+}));
+
+vi.mock('../../src/remoteGrading', () => ({
+  doRemoteGrading: mockDoRemoteGrading,
+}));
+
+vi.mock('../../src/util/fetch/index', () => ({
+  fetchWithProxy: mockFetchWithProxy,
+}));
+
+vi.mock('../../src/util/sanitizer', () => ({
+  sanitizeObject: vi.fn((obj: unknown) => obj),
+}));
+
+vi.mock('../../src/validators/util', () => ({
+  determineEffectiveSessionSource: mockDetermineEffectiveSessionSource,
+  formatConfigBody: vi.fn().mockReturnValue('None configured'),
+  formatConfigHeaders: vi.fn().mockReturnValue('None configured'),
+  validateSessionConfig: vi.fn(),
+}));
+
+import { testProviderConnectivity, testProviderSession } from '../../src/node/testProvider';
+
+beforeEach(() => {
+  // Reset call history but keep implementations from vi.hoisted
+  mockEvaluate.mockReset();
+  mockNeverGenerateRemote.mockReset().mockReturnValue(false);
+  mockDetermineEffectiveSessionSource.mockReset().mockReturnValue('client');
+  mockDoRemoteGrading.mockReset().mockResolvedValue({
+    pass: true,
+    reason: 'Session is working correctly',
+  });
+  mockToEvaluateSummary.mockReset().mockResolvedValue({
+    results: [
+      {
+        response: {
+          output: 'Hello! How can I help you?',
+          raw: 'Hello! How can I help you?',
+          sessionId: 'session-123',
+          metadata: { transformedRequest: { body: '{}' } },
+        },
+        error: undefined,
+      },
+    ],
+  });
+  mockFetchWithProxy.mockReset().mockResolvedValue({
+    ok: true,
+    json: vi.fn().mockResolvedValue({
+      message: 'Test completed successfully',
+      error: null,
+      changes_needed: false,
+    }),
+  });
+});
+
+afterEach(() => {
+  vi.clearAllMocks();
+});
+
+function createMockProvider(overrides?: Record<string, unknown>): ApiProvider {
+  return Object.assign(
+    createFactoryProvider({
+      response: {
+        output: 'Hello! How can I help you?',
+        sessionId: 'session-123',
+      },
+      config: {},
+    }),
+    overrides,
+  ) as unknown as ApiProvider;
+}
+
+describe('testProviderConnectivity', () => {
+  it('should return success when provider evaluation succeeds', async () => {
+    const provider = createMockProvider();
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(true);
+    expect(result.message).toBe('Test completed successfully');
+    expect(mockEvaluate).toHaveBeenCalled();
+  });
+
+  it('should use custom prompt when provided', async () => {
+    const provider = createMockProvider();
+    await testProviderConnectivity({ provider, prompt: 'Custom prompt' });
+
+    expect(mockEvaluate).toHaveBeenCalled();
+    const callArgs = mockEvaluate.mock.calls[0];
+    expect(callArgs[0].prompts[0].raw).toBe('Custom prompt');
+  });
+
+  it('should generate test values for input variables', async () => {
+    const provider = createMockProvider();
+    await testProviderConnectivity({
+      provider,
+      inputs: { name: 'User name', topic: 'Discussion topic' },
+    });
+
+    expect(mockEvaluate).toHaveBeenCalled();
+    const callArgs = mockEvaluate.mock.calls[0];
+    const vars = callArgs[0].tests[0].vars;
+    expect(vars['name']).toBe('test_name');
+    expect(vars['topic']).toBe('test_topic');
+  });
+
+  it('should return raw result when remote grading is disabled', async () => {
+    mockNeverGenerateRemote.mockReturnValue(true);
+    const provider = createMockProvider();
+
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(true);
+    expect(result.message).toContain('Remote grading disabled');
+    expect(mockFetchWithProxy).not.toHaveBeenCalled();
+  });
+
+  it('should return error when evaluation result has error', async () => {
+    mockNeverGenerateRemote.mockReturnValue(true);
+    mockToEvaluateSummary.mockResolvedValueOnce({
+      results: [
+        {
+          response: { output: null },
+          error: 'Connection refused',
+        },
+      ],
+    });
+
+    const provider = createMockProvider();
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('Connection refused');
+  });
+
+  it('should handle evaluation throwing an error', async () => {
+    mockEvaluate.mockRejectedValueOnce(new Error('Evaluation failed'));
+
+    const provider = createMockProvider();
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toBe('Evaluation failed');
+  });
+
+  it('should handle remote analysis endpoint failure', async () => {
+    mockFetchWithProxy.mockResolvedValueOnce({
+      ok: false,
+      statusText: 'Internal Server Error',
+    } as any);
+
+    const provider = createMockProvider();
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('review the provider response manually');
+  });
+
+  it('should handle remote analysis endpoint throwing', async () => {
+    mockFetchWithProxy.mockRejectedValueOnce(new Error('Network error'));
+
+    const provider = createMockProvider();
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toBe('Network error');
+  });
+
+  it('should report changes_needed from analysis', async () => {
+    mockFetchWithProxy.mockResolvedValueOnce({
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        message: 'Configuration needs changes',
+        error: null,
+        changes_needed: true,
+        changes_needed_reason: 'Missing auth header',
+        changes_needed_suggestions: ['Add Authorization header'],
+      }),
+    } as any);
+
+    const provider = createMockProvider();
+    const result = await testProviderConnectivity({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.analysis).toBeDefined();
+    expect(result.analysis?.changes_needed).toBe(true);
+    expect(result.analysis?.changes_needed_reason).toBe('Missing auth header');
+  });
+
+  it('should not set sessionId var when provider has sessionParser config', async () => {
+    const provider = createMockProvider({
+      config: { sessionParser: 'response.headers.session' },
+    });
+
+    await testProviderConnectivity({ provider });
+
+    expect(mockEvaluate).toHaveBeenCalled();
+    const callArgs = mockEvaluate.mock.calls[0];
+    const vars = callArgs[0].tests[0].vars;
+    expect(vars).not.toHaveProperty('sessionId');
+  });
+
+  it('should set sessionId var when provider has no sessionParser config', async () => {
+    const provider = createMockProvider();
+
+    await testProviderConnectivity({ provider });
+
+    expect(mockEvaluate).toHaveBeenCalled();
+    const callArgs = mockEvaluate.mock.calls[0];
+    const vars = callArgs[0].tests[0].vars;
+    expect(vars).toHaveProperty('sessionId');
+  });
+
+  it('should use default prompt when none provided', async () => {
+    const provider = createMockProvider();
+    await testProviderConnectivity({ provider });
+
+    const callArgs = mockEvaluate.mock.calls[0];
+    expect(callArgs[0].prompts[0].raw).toBe('Hello World!');
+  });
+});
+
+describe('testProviderSession', () => {
+  it('should return success when session is working', async () => {
+    const provider = createMockProvider();
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(true);
+    expect(result.message).toContain('working correctly');
+  });
+
+  it('should return failure when first request fails', async () => {
+    const provider = createMockProvider({
+      callApi: vi.fn().mockResolvedValue({
+        error: 'Connection timeout',
+        output: null,
+      }),
+    });
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('First request failed');
+    expect(result.error).toBe('Connection timeout');
+  });
+
+  it('should return failure when second request fails', async () => {
+    const callApi = vi
+      .fn()
+      .mockResolvedValueOnce({
+        output: 'Hello! I can help with many things.',
+        sessionId: 'session-123',
+      })
+      .mockResolvedValueOnce({
+        error: 'Server error',
+        output: null,
+      });
+
+    const provider = createMockProvider({ callApi });
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('Second request failed');
+  });
+
+  it('should return manual review message when remote grading is disabled', async () => {
+    mockNeverGenerateRemote.mockReturnValue(true);
+    const provider = createMockProvider();
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('Remote grading is disabled');
+  });
+
+  it('should return failure when session grading fails', async () => {
+    mockDoRemoteGrading.mockRejectedValueOnce(new Error('Grading service unavailable'));
+    const provider = createMockProvider();
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('Failed to evaluate session');
+  });
+
+  it('should return failure when session is not working', async () => {
+    mockDoRemoteGrading.mockResolvedValueOnce({
+      pass: false,
+      score: 0,
+      reason: 'Provider did not remember the previous question',
+    });
+    const provider = createMockProvider();
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('Session is NOT working');
+  });
+
+  it('should handle server session source with extraction failure', async () => {
+    mockDetermineEffectiveSessionSource.mockReturnValue('server');
+    const provider = createMockProvider({
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello!',
+        // No sessionId returned
+      }),
+      getSessionId: vi.fn().mockReturnValue(undefined),
+    });
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toContain('Session extraction failed');
+  });
+
+  it('should handle input variables for multi-input configurations', async () => {
+    const callApi = vi.fn().mockResolvedValue({
+      output: 'Response',
+      sessionId: 'session-123',
+    });
+    const provider = createMockProvider({ callApi });
+
+    await testProviderSession({
+      provider,
+      inputs: { user_message: 'Main input', context: 'Additional context' },
+      mainInputVariable: 'user_message',
+    });
+
+    expect(callApi).toHaveBeenCalledTimes(2);
+    const firstContext = callApi.mock.calls[0][1];
+    expect(firstContext.vars).toHaveProperty('context', 'test_context');
+    expect(firstContext.vars).toHaveProperty('user_message', 'What can you help me with?');
+  });
+
+  it('should materialize typed main input variables during session tests', async () => {
+    const callApi = vi.fn().mockResolvedValue({
+      output: 'Response',
+      sessionId: 'session-123',
+    });
+    const provider = createMockProvider({ callApi });
+
+    await testProviderSession({
+      provider,
+      inputs: {
+        document: {
+          description: 'Uploaded document',
+          type: 'docx',
+        },
+        question: 'Question',
+      },
+      mainInputVariable: 'document',
+    });
+
+    expect(callApi).toHaveBeenCalledTimes(2);
+    const firstContext = callApi.mock.calls[0][1];
+    const secondContext = callApi.mock.calls[1][1];
+    expect(firstContext.vars.document).toMatch(
+      /^data:application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document;base64,/,
+    );
+    expect(secondContext.vars.document).toMatch(
+      /^data:application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document;base64,/,
+    );
+    expect(firstContext.vars.document).not.toBe('What can you help me with?');
+    expect(secondContext.vars.document).not.toBe('What was the last thing I asked you?');
+  });
+
+  it('should handle errors thrown during session test', async () => {
+    const provider = createMockProvider({
+      callApi: vi.fn().mockRejectedValue(new Error('Unexpected error')),
+    });
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toBe('Unexpected error');
+  });
+
+  it('should include session details in successful response', async () => {
+    const provider = createMockProvider();
+    const result = await testProviderSession({ provider });
+
+    expect(result.details).toBeDefined();
+    expect(result.details?.sessionSource).toBe('client');
+    expect(result.details?.request1).toBeDefined();
+    expect(result.details?.response1).toBeDefined();
+    expect(result.details?.request2).toBeDefined();
+    expect(result.details?.response2).toBeDefined();
+  });
+
+  it('should include reason from grading judge', async () => {
+    mockDoRemoteGrading.mockResolvedValueOnce({
+      pass: true,
+      reason: 'The model remembered the first question',
+    });
+    const provider = createMockProvider();
+
+    const result = await testProviderSession({ provider });
+
+    expect(result.success).toBe(true);
+    expect(result.reason).toBe('The model remembered the first question');
+  });
+
+  it('should generate dummy values for non-main input variables', async () => {
+    const callApi = vi.fn().mockResolvedValue({
+      output: 'Response',
+      sessionId: 'session-123',
+    });
+    const provider = createMockProvider({ callApi });
+
+    await testProviderSession({
+      provider,
+      inputs: {
+        user_message: 'Main',
+        system_prompt: 'System instruction',
+        language: 'Language preference',
+      },
+      mainInputVariable: 'user_message',
+    });
+
+    const firstContext = callApi.mock.calls[0][1];
+    expect(firstContext.vars).toHaveProperty('system_prompt', 'test_system_prompt');
+    expect(firstContext.vars).toHaveProperty('language', 'test_language');
+    // main input variable should have the actual prompt
+    expect(firstContext.vars).toHaveProperty('user_message', 'What can you help me with?');
+  });
+});