import { Command } from 'commander'; import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest'; import { listCommand } from '../../src/commands/list'; import logger from '../../src/logger'; import Eval, { EvalQueries } from '../../src/models/eval'; import { wrapTable } from '../../src/table'; import { sha256 } from '../../src/util/createHash'; import { getPrompts, getTestCases } from '../../src/util/database'; import { printBorder, setupEnv } from '../../src/util/index'; import type { PromptWithMetadata, TestCasesWithMetadata } from '../../src/types/index'; vi.mock('../../src/logger', () => ({ default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), }, })); vi.mock('../../src/models/eval', () => ({ default: { getMany: vi.fn(), }, EvalQueries: { getVarsFromEvals: vi.fn(), }, })); vi.mock('../../src/table', () => ({ wrapTable: vi.fn(), })); vi.mock('../../src/util/createHash', () => ({ sha256: vi.fn(), })); vi.mock('../../src/util/database', () => ({ getPrompts: vi.fn(), getTestCases: vi.fn(), })); vi.mock('../../src/util/index', () => ({ printBorder: vi.fn(), setupEnv: vi.fn(), })); describe('list command', () => { let program: Command; const evalModel = Eval as unknown as { getMany: Mock; }; const evalQueries = EvalQueries as unknown as { getVarsFromEvals: Mock; }; function getListSubcommand(name: 'evals' | 'prompts' | 'datasets') { const listCmd = program.commands.find((cmd) => cmd.name() === 'list'); return listCmd?.commands.find((cmd) => cmd.name() === name); } function createEval({ id, createdAt, description, providers, prompts, }: { id: string; createdAt: number; description?: string; providers: unknown; prompts: Array<{ raw: string; metrics?: { score?: number; testPassCount?: number; testFailCount?: number; testErrorCount?: number; }; }>; }) { return { id, createdAt, config: { description, providers, }, getPrompts: vi.fn().mockReturnValue(prompts), }; } beforeEach(() => { vi.resetAllMocks(); process.exitCode = 0; program = new Command(); listCommand(program); vi.mocked(wrapTable).mockReturnValue('mocked table'); vi.mocked(sha256).mockImplementation((input: string | Buffer) => `hash-${input}`); vi.mocked(getPrompts).mockResolvedValue([]); vi.mocked(getTestCases).mockResolvedValue([]); evalModel.getMany.mockResolvedValue([]); evalQueries.getVarsFromEvals.mockResolvedValue({}); }); afterEach(() => { vi.resetAllMocks(); }); describe('registration', () => { it('registers list command and subcommands', () => { const listCmd = program.commands.find((cmd) => cmd.name() === 'list'); expect(listCmd).toBeDefined(); expect(listCmd?.description()).toBe('List various resources'); expect(listCmd?.commands.map((cmd) => cmd.name())).toEqual( expect.arrayContaining(['evals', 'prompts', 'datasets']), ); }); }); describe('evals', () => { it('prints eval ids with --ids-only', async () => { evalModel.getMany.mockResolvedValue([{ id: 'eval-1' }, { id: 'eval-2' }]); const evalsCmd = getListSubcommand('evals'); await evalsCmd?.parseAsync(['node', 'test', '--ids-only', '-n', '2', '--env-path', '.env']); expect(setupEnv).toHaveBeenCalledWith('.env'); expect(evalModel.getMany).toHaveBeenCalledWith(2); expect(logger.info).toHaveBeenNthCalledWith(1, 'eval-1'); expect(logger.info).toHaveBeenNthCalledWith(2, 'eval-2'); expect(wrapTable).not.toHaveBeenCalled(); }); it('renders table output', async () => { const olderEval = createEval({ id: 'eval-old', createdAt: 1, description: 'Older evaluation', providers: ['openai:gpt-4'], prompts: [{ raw: 'Older prompt' }], }); const newerEval = createEval({ id: 'eval-new', createdAt: 2, description: 'Newer evaluation', providers: ['openai:gpt-4'], prompts: [{ raw: 'Newer prompt' }], }); evalModel.getMany.mockResolvedValue([newerEval, olderEval]); evalQueries.getVarsFromEvals.mockResolvedValue({ 'eval-old': ['topic'], 'eval-new': ['name', 'city'], }); const evalsCmd = getListSubcommand('evals'); await evalsCmd?.parseAsync(['node', 'test', '-n', '2']); expect(evalModel.getMany).toHaveBeenCalledWith(2); // Note: evals.sort() in list.ts mutates the array in place, so the mock // reference reflects the sorted (ascending createdAt) order by assertion time. expect(evalQueries.getVarsFromEvals).toHaveBeenCalledWith([olderEval, newerEval]); expect(wrapTable).toHaveBeenCalledTimes(1); const [tableRows] = vi.mocked(wrapTable).mock.calls[0]; expect(tableRows[0]).toMatchObject({ 'eval id': 'eval-old', prompts: 'hash-O', vars: 'topic', }); expect(tableRows[1]).toMatchObject({ 'eval id': 'eval-new', prompts: 'hash-N', vars: 'name, city', }); expect(printBorder).toHaveBeenCalledTimes(1); expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('promptfoo show eval ')); expect(logger.info).toHaveBeenCalledWith( expect.stringContaining('promptfoo show prompt '), ); }); }); describe('prompts', () => { // Prompts are sorted by recentEvalId (ascending), so eval-a < eval-b it('prints prompt ids with --ids-only in sorted order', async () => { const prompts: PromptWithMetadata[] = [ { id: 'prompt-b', prompt: { raw: 'Prompt B', label: 'Prompt B' }, count: 1, recentEvalDate: new Date('2025-01-01T00:00:00.000Z'), recentEvalId: 'eval-b', evals: [], }, { id: 'prompt-a', prompt: { raw: 'Prompt A', label: 'Prompt A' }, count: 2, recentEvalDate: new Date('2025-01-02T00:00:00.000Z'), recentEvalId: 'eval-a', evals: [], }, ]; vi.mocked(getPrompts).mockResolvedValue(prompts); const promptsCmd = getListSubcommand('prompts'); await promptsCmd?.parseAsync(['node', 'test', '--ids-only', '--env-path', '.env.prompts']); expect(setupEnv).toHaveBeenCalledWith('.env.prompts'); expect(logger.info).toHaveBeenNthCalledWith(1, 'prompt-a'); expect(logger.info).toHaveBeenNthCalledWith(2, 'prompt-b'); }); it('renders prompts table output', async () => { const prompts: PromptWithMetadata[] = [ { id: 'abcdef123456', prompt: { raw: 'x'.repeat(110), label: 'x'.repeat(110) }, count: 3, recentEvalDate: new Date('2025-01-03T00:00:00.000Z'), recentEvalId: 'eval-123', evals: [], }, ]; vi.mocked(getPrompts).mockResolvedValue(prompts); const promptsCmd = getListSubcommand('prompts'); await promptsCmd?.parseAsync(['node', 'test', '-n', '1']); expect(getPrompts).toHaveBeenCalledWith(1); expect(wrapTable).toHaveBeenCalledTimes(1); const [tableRows] = vi.mocked(wrapTable).mock.calls[0]; expect(tableRows[0]).toMatchObject({ 'prompt id': 'abcdef', evals: 3, 'recent eval': 'eval-123', }); expect(tableRows[0].raw).toMatch(/\.\.\.$/); expect(printBorder).toHaveBeenCalledTimes(1); expect(logger.info).toHaveBeenCalledWith( expect.stringContaining('promptfoo show prompt '), ); }); }); describe('datasets', () => { // Datasets are sorted by recentEvalId (descending), so eval-999 > eval-001 it('prints dataset ids with --ids-only in sorted order', async () => { const datasets: TestCasesWithMetadata[] = [ { id: 'dataset-old', testCases: [], prompts: [], count: 1, recentEvalDate: new Date('2025-01-01T00:00:00.000Z'), recentEvalId: 'eval-001', }, { id: 'dataset-new', testCases: [], prompts: [], count: 2, recentEvalDate: new Date('2025-01-02T00:00:00.000Z'), recentEvalId: 'eval-999', }, ]; vi.mocked(getTestCases).mockResolvedValue(datasets); const datasetsCmd = getListSubcommand('datasets'); await datasetsCmd?.parseAsync(['node', 'test', '--ids-only']); expect(logger.info).toHaveBeenNthCalledWith(1, 'dataset-new'); expect(logger.info).toHaveBeenNthCalledWith(2, 'dataset-old'); expect(wrapTable).not.toHaveBeenCalled(); }); it('renders the best prompt from the highest-scoring dataset prompt', async () => { const datasets: TestCasesWithMetadata[] = [ { id: 'dataset-1', testCases: [], count: 3, recentEvalDate: new Date('2025-01-10T00:00:00.000Z'), recentEvalId: 'eval-10', prompts: [ { id: 'low-001', evalId: 'eval-10', prompt: { raw: 'Low', label: 'Low', provider: 'echo', metrics: { score: 0.3, testPassCount: 0, testFailCount: 0, testErrorCount: 0, assertPassCount: 0, assertFailCount: 0, totalLatencyMs: 0, tokenUsage: {}, namedScores: {}, namedScoresCount: {}, cost: 0, }, }, }, { id: 'high-001', evalId: 'eval-10', prompt: { raw: 'High', label: 'High', provider: 'echo', metrics: { score: 0.9, testPassCount: 0, testFailCount: 0, testErrorCount: 0, assertPassCount: 0, assertFailCount: 0, totalLatencyMs: 0, tokenUsage: {}, namedScores: {}, namedScoresCount: {}, cost: 0, }, }, }, ], }, ]; const datasetId = datasets[0].id; const bestPromptId = datasets[0].prompts[1].id; vi.mocked(getTestCases).mockResolvedValue(datasets); const datasetsCmd = getListSubcommand('datasets'); await datasetsCmd?.parseAsync(['node', 'test']); expect(wrapTable).toHaveBeenCalledTimes(1); const [tableRows] = vi.mocked(wrapTable).mock.calls[0]; expect(tableRows[0]).toMatchObject({ 'dataset id': datasetId.slice(0, 6), 'best prompt': bestPromptId.slice(0, 6), evals: 3, prompts: 2, 'recent eval': 'eval-10', }); }); }); describe('error handling', () => { it('propagates error when Eval.getMany rejects in evals --ids-only', async () => { evalModel.getMany.mockRejectedValue(new Error('Database error')); const evalsCmd = getListSubcommand('evals'); await expect(evalsCmd?.parseAsync(['node', 'test', '--ids-only'])).rejects.toThrow( 'Database error', ); }); it('propagates error when getPrompts rejects', async () => { vi.mocked(getPrompts).mockRejectedValue(new Error('Prompts fetch failed')); const promptsCmd = getListSubcommand('prompts'); await expect(promptsCmd?.parseAsync(['node', 'test', '--ids-only'])).rejects.toThrow( 'Prompts fetch failed', ); }); it('propagates error when getTestCases rejects', async () => { vi.mocked(getTestCases).mockRejectedValue(new Error('Datasets fetch failed')); const datasetsCmd = getListSubcommand('datasets'); await expect(datasetsCmd?.parseAsync(['node', 'test', '--ids-only'])).rejects.toThrow( 'Datasets fetch failed', ); }); }); });