wehub-resource-sync
0d3cb498a3
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
892 lines
26 KiB
TypeScript
892 lines
26 KiB
TypeScript
import chalk from 'chalk';
|
|
import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
|
|
import { generateEvalSummary } from '../../../src/util/eval/summary';
|
|
import { stripAnsi } from '../../util/utils';
|
|
|
|
import type { EvalSummaryParams } from '../../../src/util/eval/summary';
|
|
import type { TokenUsageTracker } from '../../../src/util/tokenUsage';
|
|
|
|
type MockTracker = {
|
|
getProviderIds: Mock;
|
|
getProviderUsage: Mock;
|
|
trackUsage: Mock;
|
|
resetAllUsage: Mock;
|
|
resetProviderUsage: Mock;
|
|
getTotalUsage: Mock;
|
|
cleanup: Mock;
|
|
};
|
|
|
|
function createMockTracker(): TokenUsageTracker {
|
|
return {
|
|
getProviderIds: vi.fn().mockReturnValue([]),
|
|
getProviderUsage: vi.fn(),
|
|
trackUsage: vi.fn(),
|
|
resetAllUsage: vi.fn(),
|
|
resetProviderUsage: vi.fn(),
|
|
getTotalUsage: vi.fn(),
|
|
cleanup: vi.fn(),
|
|
} as unknown as TokenUsageTracker;
|
|
}
|
|
|
|
describe('generateEvalSummary', () => {
|
|
let mockTracker: MockTracker & TokenUsageTracker;
|
|
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
mockTracker = createMockTracker() as MockTracker & TokenUsageTracker;
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
describe('completion message', () => {
|
|
it('should show basic completion message when not writing to database', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-123',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('✓ Eval complete');
|
|
expect(output).not.toContain('eval-123');
|
|
});
|
|
|
|
it('should show eval ID when writing to database without shareable URL', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-456',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('✓ Eval complete (ID: eval-456)');
|
|
});
|
|
|
|
it('should show shareable URL when available', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-789',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: 'https://promptfoo.app/eval/abc123',
|
|
wantsToShare: true,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('✓ Eval complete: https://promptfoo.app/eval/abc123');
|
|
expect(output).not.toContain('eval-789');
|
|
});
|
|
|
|
it('should say "Red team complete" for red team evals', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-rt-1',
|
|
isRedteam: true,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 10,
|
|
failures: 2,
|
|
errors: 0,
|
|
duration: 8000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('✓ Red team complete');
|
|
expect(output).not.toContain('Eval complete');
|
|
});
|
|
|
|
it('should explain non-retryable target errors when an eval is aborted', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-aborted',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 0,
|
|
failures: 0,
|
|
errors: 1,
|
|
duration: 1000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
targetErrorStatus: 401,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('✗ Eval aborted (ID: eval-aborted)');
|
|
expect(output).toContain(
|
|
'Scan stopped: Target is unavailable and will not recover on retry.',
|
|
);
|
|
expect(output).toContain('Target returned HTTP 401');
|
|
expect(output).not.toContain('Server error (500)');
|
|
expect(output).toContain('To fix: Check your target configuration and credentials.');
|
|
});
|
|
});
|
|
|
|
describe('token usage', () => {
|
|
it('should display eval tokens correctly', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-123',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: {
|
|
total: 1000,
|
|
prompt: 400,
|
|
completion: 600,
|
|
cached: 0,
|
|
},
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Tokens:');
|
|
expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion)');
|
|
});
|
|
|
|
it('should display grading tokens only when no eval tokens (critical bug fix)', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-grading-only',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: {
|
|
total: 0,
|
|
assertions: {
|
|
total: 500,
|
|
prompt: 200,
|
|
completion: 300,
|
|
cached: 0,
|
|
},
|
|
},
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Tokens:');
|
|
expect(output).toContain('Grading: 500 (200 prompt, 300 completion)');
|
|
expect(output).not.toContain('Eval:');
|
|
});
|
|
|
|
it('should display both eval and grading tokens', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-both',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: {
|
|
total: 1000,
|
|
prompt: 400,
|
|
completion: 600,
|
|
assertions: {
|
|
total: 500,
|
|
prompt: 200,
|
|
completion: 300,
|
|
},
|
|
},
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Tokens:');
|
|
expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion)');
|
|
expect(output).toContain('Grading: 500 (200 prompt, 300 completion)');
|
|
});
|
|
|
|
it('should show 100% cached correctly', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-cached',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: {
|
|
total: 1000,
|
|
cached: 1000,
|
|
},
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Tokens:');
|
|
expect(output).toContain('Eval: 1,000 (cached)');
|
|
});
|
|
|
|
it('should show partial cached tokens', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-partial-cache',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: {
|
|
total: 1000,
|
|
prompt: 400,
|
|
completion: 600,
|
|
cached: 200,
|
|
},
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Eval: 1,000 (400 prompt, 600 completion, 200 cached)');
|
|
});
|
|
|
|
it('should not show token section when no tokens', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-no-tokens',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).not.toContain('Tokens:');
|
|
});
|
|
});
|
|
|
|
describe('provider breakdown', () => {
|
|
it('should show provider breakdown with request counts', () => {
|
|
mockTracker.getProviderIds.mockReturnValue(['openai:gpt-4', 'anthropic:claude-3']);
|
|
mockTracker.getProviderUsage.mockImplementation((id: string) => {
|
|
if (id === 'openai:gpt-4') {
|
|
return {
|
|
total: 1500,
|
|
prompt: 600,
|
|
completion: 900,
|
|
cached: 0,
|
|
numRequests: 5,
|
|
};
|
|
}
|
|
if (id === 'anthropic:claude-3') {
|
|
return {
|
|
total: 800,
|
|
prompt: 300,
|
|
completion: 500,
|
|
cached: 0,
|
|
numRequests: 3,
|
|
};
|
|
}
|
|
return undefined;
|
|
});
|
|
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-providers',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 2300 },
|
|
successes: 8,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Providers:');
|
|
expect(output).toContain('openai:gpt-4');
|
|
expect(output).toContain('1,500 (5 requests; 600 prompt, 900 completion)');
|
|
expect(output).toContain('anthropic:claude-3');
|
|
expect(output).toContain('800 (3 requests; 300 prompt, 500 completion)');
|
|
});
|
|
|
|
it('should always show request count even when 0', () => {
|
|
mockTracker.getProviderIds.mockReturnValue(['openai:gpt-4', 'anthropic:claude-3']);
|
|
mockTracker.getProviderUsage.mockImplementation((id: string) => {
|
|
if (id === 'openai:gpt-4') {
|
|
return {
|
|
total: 1000,
|
|
cached: 1000,
|
|
numRequests: 0,
|
|
};
|
|
}
|
|
if (id === 'anthropic:claude-3') {
|
|
return {
|
|
total: 500,
|
|
prompt: 200,
|
|
completion: 300,
|
|
numRequests: 2,
|
|
};
|
|
}
|
|
return undefined;
|
|
});
|
|
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-zero-requests',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 1500 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Providers:');
|
|
expect(output).toContain('openai:gpt-4: 1,000 (0 requests; cached)');
|
|
expect(output).toContain('anthropic:claude-3: 500 (2 requests; 200 prompt, 300 completion)');
|
|
});
|
|
});
|
|
|
|
describe('pass rate and results', () => {
|
|
it('should show percentages for each result line at 100% pass rate', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-100',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 10,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
|
|
expect(plainOutput).toContain('Results:');
|
|
expect(plainOutput).toContain('10 passed (100%)');
|
|
expect(plainOutput).toContain('0 failed (0%)');
|
|
expect(plainOutput).toContain('0 errors (0%)');
|
|
});
|
|
|
|
it('should show percentages for each result line when some tests fail', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-85',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 17,
|
|
failures: 3,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
|
|
expect(plainOutput).toContain('Results:');
|
|
expect(plainOutput).toContain('17 passed (85.00%)');
|
|
expect(plainOutput).toContain('3 failed (15.00%)');
|
|
expect(plainOutput).toContain('0 errors (0%)');
|
|
});
|
|
|
|
it('should show percentages for each result line when passed and failed are split evenly', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-50',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 5,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
|
|
expect(plainOutput).toContain('Results:');
|
|
expect(plainOutput).toContain('5 passed (50.00%)');
|
|
expect(plainOutput).toContain('5 failed (50.00%)');
|
|
expect(plainOutput).toContain('0 errors (0%)');
|
|
});
|
|
|
|
it('should include errors in results', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-errors',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 8,
|
|
failures: 1,
|
|
errors: 1,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
const outputLines = plainOutput.split('\n');
|
|
const hasLineMatching = (pattern: RegExp) => outputLines.some((line) => pattern.test(line));
|
|
|
|
expect(plainOutput).toContain('Results:');
|
|
expect(hasLineMatching(/^\s*(✓\s+)?8 passed \(80\.00%\)$/)).toBe(true);
|
|
expect(hasLineMatching(/^\s*(✗\s+)?1 failed \(10\.00%\)$/)).toBe(true);
|
|
expect(hasLineMatching(/^\s*(✗\s+)?1 error \(10\.00%\)$/)).toBe(true);
|
|
expect(plainOutput).not.toContain('1 errors');
|
|
});
|
|
|
|
it('should render colored icons with muted percentages', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-styling',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 8,
|
|
failures: 1,
|
|
errors: 1,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
|
|
expect(lines).toContain(
|
|
` ${chalk.green('✓')} ${chalk.white.bold('8')} ${chalk.white('passed')} ${chalk.gray('(80.00%)')}`,
|
|
);
|
|
expect(lines).toContain(
|
|
` ${chalk.red('✗')} ${chalk.white.bold('1')} ${chalk.white('failed')} ${chalk.gray('(10.00%)')}`,
|
|
);
|
|
expect(lines).toContain(
|
|
` ${chalk.red('✗')} ${chalk.white.bold('1')} ${chalk.white('error')} ${chalk.gray('(10.00%)')}`,
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('guidance messages', () => {
|
|
it('should show guidance when writing to database without shareable URL', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-view',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('» View results: promptfoo view');
|
|
expect(output).toContain('» Share with your team: https://promptfoo.app');
|
|
expect(output).toContain('» Feedback: https://promptfoo.dev/feedback');
|
|
});
|
|
|
|
it('should show share guidance with cloud enabled', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-share-cloud',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: true,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('» View results: promptfoo view');
|
|
expect(output).toContain('» Create shareable URL: promptfoo share');
|
|
expect(output).not.toContain('https://promptfoo.app');
|
|
});
|
|
|
|
it('should NOT show share guidance when explicitly disabled (--no-share)', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-no-share',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: true,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('» View results: promptfoo view');
|
|
expect(output).not.toContain('» Share with your team');
|
|
expect(output).not.toContain('» Create shareable URL');
|
|
});
|
|
|
|
it('should NOT show guidance when not writing to database', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-no-write',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).not.toContain('» View results:');
|
|
expect(output).not.toContain('» Share');
|
|
expect(output).not.toContain('» Feedback:');
|
|
});
|
|
|
|
it('should NOT show guidance when shareable URL is present', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-with-url',
|
|
isRedteam: false,
|
|
writeToDatabase: true,
|
|
shareableUrl: 'https://promptfoo.app/eval/abc123',
|
|
wantsToShare: true,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: true,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).not.toContain('» View results:');
|
|
expect(output).not.toContain('» Share');
|
|
expect(output).not.toContain('» Feedback:');
|
|
});
|
|
});
|
|
|
|
describe('performance metrics', () => {
|
|
it('should show duration and concurrency', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-perf',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 125, // 125 seconds = 2m 5s
|
|
maxConcurrency: 8,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
expect(output).toContain('Duration:');
|
|
expect(output).toContain('(concurrency: 8)');
|
|
});
|
|
});
|
|
|
|
describe('edge cases', () => {
|
|
it('should use singular "error" when there is exactly 1 error', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-singular-error',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 1,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
|
|
expect(plainOutput).toContain('1 error');
|
|
expect(plainOutput).not.toContain('1 errors');
|
|
});
|
|
|
|
it('should use plural "errors" when there are multiple errors', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-plural-errors',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 3,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
|
|
expect(plainOutput).toContain('3 errors');
|
|
});
|
|
|
|
it('should use plural "errors" when there are 0 errors', () => {
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-zero-errors',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 0 },
|
|
successes: 5,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
const lines = generateEvalSummary(params);
|
|
const plainOutput = stripAnsi(lines.join('\n'));
|
|
|
|
expect(plainOutput).toContain('0 errors');
|
|
});
|
|
|
|
it('should handle provider returning undefined usage gracefully', () => {
|
|
mockTracker.getProviderIds.mockReturnValue([
|
|
'openai:gpt-4',
|
|
'missing-provider',
|
|
'anthropic:claude-3',
|
|
]);
|
|
mockTracker.getProviderUsage.mockImplementation((id: string) => {
|
|
if (id === 'openai:gpt-4') {
|
|
return {
|
|
total: 1000,
|
|
prompt: 400,
|
|
completion: 600,
|
|
numRequests: 5,
|
|
};
|
|
}
|
|
if (id === 'missing-provider') {
|
|
return undefined; // Simulates a provider that returns undefined
|
|
}
|
|
if (id === 'anthropic:claude-3') {
|
|
return {
|
|
total: 500,
|
|
prompt: 200,
|
|
completion: 300,
|
|
numRequests: 3,
|
|
};
|
|
}
|
|
return undefined;
|
|
});
|
|
|
|
const params: EvalSummaryParams = {
|
|
evalId: 'eval-undefined-provider',
|
|
isRedteam: false,
|
|
writeToDatabase: false,
|
|
shareableUrl: null,
|
|
wantsToShare: false,
|
|
hasExplicitDisable: false,
|
|
cloudEnabled: false,
|
|
tokenUsage: { total: 1500 },
|
|
successes: 8,
|
|
failures: 0,
|
|
errors: 0,
|
|
duration: 5000,
|
|
maxConcurrency: 4,
|
|
tracker: mockTracker,
|
|
};
|
|
|
|
// Should not throw
|
|
const lines = generateEvalSummary(params);
|
|
const output = stripAnsi(lines.join('\n'));
|
|
|
|
// Should show the providers that have valid usage
|
|
expect(output).toContain('Providers:');
|
|
expect(output).toContain('openai:gpt-4');
|
|
expect(output).toContain('anthropic:claude-3');
|
|
// Should NOT show the missing provider
|
|
expect(output).not.toContain('missing-provider');
|
|
});
|
|
});
|
|
});
|