promptfoo--promptfoo/test/commands/list.test.ts

import { Command } from 'commander';
import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
import { listCommand } from '../../src/commands/list';
import logger from '../../src/logger';
import Eval, { EvalQueries } from '../../src/models/eval';
import { wrapTable } from '../../src/table';
import { sha256 } from '../../src/util/createHash';
import { getPrompts, getTestCases } from '../../src/util/database';
import { printBorder, setupEnv } from '../../src/util/index';

import type { PromptWithMetadata, TestCasesWithMetadata } from '../../src/types/index';

vi.mock('../../src/logger', () => ({
  default: {
    info: vi.fn(),
    warn: vi.fn(),
    error: vi.fn(),
    debug: vi.fn(),
  },
}));

vi.mock('../../src/models/eval', () => ({
  default: {
    getMany: vi.fn(),
  },
  EvalQueries: {
    getVarsFromEvals: vi.fn(),
  },
}));

vi.mock('../../src/table', () => ({
  wrapTable: vi.fn(),
}));

vi.mock('../../src/util/createHash', () => ({
  sha256: vi.fn(),
}));

vi.mock('../../src/util/database', () => ({
  getPrompts: vi.fn(),
  getTestCases: vi.fn(),
}));

vi.mock('../../src/util/index', () => ({
  printBorder: vi.fn(),
  setupEnv: vi.fn(),
}));

describe('list command', () => {
  let program: Command;
  const evalModel = Eval as unknown as {
    getMany: Mock;
  };
  const evalQueries = EvalQueries as unknown as {
    getVarsFromEvals: Mock;
  };

  function getListSubcommand(name: 'evals' | 'prompts' | 'datasets') {
    const listCmd = program.commands.find((cmd) => cmd.name() === 'list');
    return listCmd?.commands.find((cmd) => cmd.name() === name);
  }

  function createEval({
    id,
    createdAt,
    description,
    providers,
    prompts,
  }: {
    id: string;
    createdAt: number;
    description?: string;
    providers: unknown;
    prompts: Array<{
      raw: string;
      metrics?: {
        score?: number;
        testPassCount?: number;
        testFailCount?: number;
        testErrorCount?: number;
      };
    }>;
  }) {
    return {
      id,
      createdAt,
      config: {
        description,
        providers,
      },
      getPrompts: vi.fn().mockReturnValue(prompts),
    };
  }

  beforeEach(() => {
    vi.resetAllMocks();
    process.exitCode = 0;
    program = new Command();
    listCommand(program);
    vi.mocked(wrapTable).mockReturnValue('mocked table');
    vi.mocked(sha256).mockImplementation((input: string | Buffer) => `hash-${input}`);
    vi.mocked(getPrompts).mockResolvedValue([]);
    vi.mocked(getTestCases).mockResolvedValue([]);
    evalModel.getMany.mockResolvedValue([]);
    evalQueries.getVarsFromEvals.mockResolvedValue({});
  });

  afterEach(() => {
    vi.resetAllMocks();
  });

  describe('registration', () => {
    it('registers list command and subcommands', () => {
      const listCmd = program.commands.find((cmd) => cmd.name() === 'list');

      expect(listCmd).toBeDefined();
      expect(listCmd?.description()).toBe('List various resources');
      expect(listCmd?.commands.map((cmd) => cmd.name())).toEqual(
        expect.arrayContaining(['evals', 'prompts', 'datasets']),
      );
    });
  });

  describe('evals', () => {
    it('prints eval ids with --ids-only', async () => {
      evalModel.getMany.mockResolvedValue([{ id: 'eval-1' }, { id: 'eval-2' }]);

      const evalsCmd = getListSubcommand('evals');
      await evalsCmd?.parseAsync(['node', 'test', '--ids-only', '-n', '2', '--env-path', '.env']);

      expect(setupEnv).toHaveBeenCalledWith('.env');
      expect(evalModel.getMany).toHaveBeenCalledWith(2);
      expect(logger.info).toHaveBeenNthCalledWith(1, 'eval-1');
      expect(logger.info).toHaveBeenNthCalledWith(2, 'eval-2');
      expect(wrapTable).not.toHaveBeenCalled();
    });

    it('renders table output', async () => {
      const olderEval = createEval({
        id: 'eval-old',
        createdAt: 1,
        description: 'Older evaluation',
        providers: ['openai:gpt-4'],
        prompts: [{ raw: 'Older prompt' }],
      });
      const newerEval = createEval({
        id: 'eval-new',
        createdAt: 2,
        description: 'Newer evaluation',
        providers: ['openai:gpt-4'],
        prompts: [{ raw: 'Newer prompt' }],
      });
      evalModel.getMany.mockResolvedValue([newerEval, olderEval]);
      evalQueries.getVarsFromEvals.mockResolvedValue({
        'eval-old': ['topic'],
        'eval-new': ['name', 'city'],
      });

      const evalsCmd = getListSubcommand('evals');
      await evalsCmd?.parseAsync(['node', 'test', '-n', '2']);

      expect(evalModel.getMany).toHaveBeenCalledWith(2);
      // Note: evals.sort() in list.ts mutates the array in place, so the mock
      // reference reflects the sorted (ascending createdAt) order by assertion time.
      expect(evalQueries.getVarsFromEvals).toHaveBeenCalledWith([olderEval, newerEval]);
      expect(wrapTable).toHaveBeenCalledTimes(1);

      const [tableRows] = vi.mocked(wrapTable).mock.calls[0];
      expect(tableRows[0]).toMatchObject({
        'eval id': 'eval-old',
        prompts: 'hash-O',
        vars: 'topic',
      });
      expect(tableRows[1]).toMatchObject({
        'eval id': 'eval-new',
        prompts: 'hash-N',
        vars: 'name, city',
      });

      expect(printBorder).toHaveBeenCalledTimes(1);
      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('promptfoo show eval <id>'));
      expect(logger.info).toHaveBeenCalledWith(
        expect.stringContaining('promptfoo show prompt <id>'),
      );
    });
  });

  describe('prompts', () => {
    // Prompts are sorted by recentEvalId (ascending), so eval-a < eval-b
    it('prints prompt ids with --ids-only in sorted order', async () => {
      const prompts: PromptWithMetadata[] = [
        {
          id: 'prompt-b',
          prompt: { raw: 'Prompt B', label: 'Prompt B' },
          count: 1,
          recentEvalDate: new Date('2025-01-01T00:00:00.000Z'),
          recentEvalId: 'eval-b',
          evals: [],
        },
        {
          id: 'prompt-a',
          prompt: { raw: 'Prompt A', label: 'Prompt A' },
          count: 2,
          recentEvalDate: new Date('2025-01-02T00:00:00.000Z'),
          recentEvalId: 'eval-a',
          evals: [],
        },
      ];
      vi.mocked(getPrompts).mockResolvedValue(prompts);

      const promptsCmd = getListSubcommand('prompts');
      await promptsCmd?.parseAsync(['node', 'test', '--ids-only', '--env-path', '.env.prompts']);

      expect(setupEnv).toHaveBeenCalledWith('.env.prompts');
      expect(logger.info).toHaveBeenNthCalledWith(1, 'prompt-a');
      expect(logger.info).toHaveBeenNthCalledWith(2, 'prompt-b');
    });

    it('renders prompts table output', async () => {
      const prompts: PromptWithMetadata[] = [
        {
          id: 'abcdef123456',
          prompt: { raw: 'x'.repeat(110), label: 'x'.repeat(110) },
          count: 3,
          recentEvalDate: new Date('2025-01-03T00:00:00.000Z'),
          recentEvalId: 'eval-123',
          evals: [],
        },
      ];
      vi.mocked(getPrompts).mockResolvedValue(prompts);

      const promptsCmd = getListSubcommand('prompts');
      await promptsCmd?.parseAsync(['node', 'test', '-n', '1']);

      expect(getPrompts).toHaveBeenCalledWith(1);
      expect(wrapTable).toHaveBeenCalledTimes(1);
      const [tableRows] = vi.mocked(wrapTable).mock.calls[0];
      expect(tableRows[0]).toMatchObject({
        'prompt id': 'abcdef',
        evals: 3,
        'recent eval': 'eval-123',
      });
      expect(tableRows[0].raw).toMatch(/\.\.\.$/);
      expect(printBorder).toHaveBeenCalledTimes(1);
      expect(logger.info).toHaveBeenCalledWith(
        expect.stringContaining('promptfoo show prompt <id>'),
      );
    });
  });

  describe('datasets', () => {
    // Datasets are sorted by recentEvalId (descending), so eval-999 > eval-001
    it('prints dataset ids with --ids-only in sorted order', async () => {
      const datasets: TestCasesWithMetadata[] = [
        {
          id: 'dataset-old',
          testCases: [],
          prompts: [],
          count: 1,
          recentEvalDate: new Date('2025-01-01T00:00:00.000Z'),
          recentEvalId: 'eval-001',
        },
        {
          id: 'dataset-new',
          testCases: [],
          prompts: [],
          count: 2,
          recentEvalDate: new Date('2025-01-02T00:00:00.000Z'),
          recentEvalId: 'eval-999',
        },
      ];
      vi.mocked(getTestCases).mockResolvedValue(datasets);

      const datasetsCmd = getListSubcommand('datasets');
      await datasetsCmd?.parseAsync(['node', 'test', '--ids-only']);

      expect(logger.info).toHaveBeenNthCalledWith(1, 'dataset-new');
      expect(logger.info).toHaveBeenNthCalledWith(2, 'dataset-old');
      expect(wrapTable).not.toHaveBeenCalled();
    });

    it('renders the best prompt from the highest-scoring dataset prompt', async () => {
      const datasets: TestCasesWithMetadata[] = [
        {
          id: 'dataset-1',
          testCases: [],
          count: 3,
          recentEvalDate: new Date('2025-01-10T00:00:00.000Z'),
          recentEvalId: 'eval-10',
          prompts: [
            {
              id: 'low-001',
              evalId: 'eval-10',
              prompt: {
                raw: 'Low',
                label: 'Low',
                provider: 'echo',
                metrics: {
                  score: 0.3,
                  testPassCount: 0,
                  testFailCount: 0,
                  testErrorCount: 0,
                  assertPassCount: 0,
                  assertFailCount: 0,
                  totalLatencyMs: 0,
                  tokenUsage: {},
                  namedScores: {},
                  namedScoresCount: {},
                  cost: 0,
                },
              },
            },
            {
              id: 'high-001',
              evalId: 'eval-10',
              prompt: {
                raw: 'High',
                label: 'High',
                provider: 'echo',
                metrics: {
                  score: 0.9,
                  testPassCount: 0,
                  testFailCount: 0,
                  testErrorCount: 0,
                  assertPassCount: 0,
                  assertFailCount: 0,
                  totalLatencyMs: 0,
                  tokenUsage: {},
                  namedScores: {},
                  namedScoresCount: {},
                  cost: 0,
                },
              },
            },
          ],
        },
      ];
      const datasetId = datasets[0].id;
      const bestPromptId = datasets[0].prompts[1].id;
      vi.mocked(getTestCases).mockResolvedValue(datasets);

      const datasetsCmd = getListSubcommand('datasets');
      await datasetsCmd?.parseAsync(['node', 'test']);

      expect(wrapTable).toHaveBeenCalledTimes(1);
      const [tableRows] = vi.mocked(wrapTable).mock.calls[0];
      expect(tableRows[0]).toMatchObject({
        'dataset id': datasetId.slice(0, 6),
        'best prompt': bestPromptId.slice(0, 6),
        evals: 3,
        prompts: 2,
        'recent eval': 'eval-10',
      });
    });
  });

  describe('error handling', () => {
    it('propagates error when Eval.getMany rejects in evals --ids-only', async () => {
      evalModel.getMany.mockRejectedValue(new Error('Database error'));

      const evalsCmd = getListSubcommand('evals');
      await expect(evalsCmd?.parseAsync(['node', 'test', '--ids-only'])).rejects.toThrow(
        'Database error',
      );
    });

    it('propagates error when getPrompts rejects', async () => {
      vi.mocked(getPrompts).mockRejectedValue(new Error('Prompts fetch failed'));

      const promptsCmd = getListSubcommand('prompts');
      await expect(promptsCmd?.parseAsync(['node', 'test', '--ids-only'])).rejects.toThrow(
        'Prompts fetch failed',
      );
    });

    it('propagates error when getTestCases rejects', async () => {
      vi.mocked(getTestCases).mockRejectedValue(new Error('Datasets fetch failed'));

      const datasetsCmd = getListSubcommand('datasets');
      await expect(datasetsCmd?.parseAsync(['node', 'test', '--ids-only'])).rejects.toThrow(
        'Datasets fetch failed',
      );
    });
  });
});