wehub-resource-sync
0d3cb498a3
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Waiting to run
Test and Publish Multi-arch Docker Image / test (push) Waiting to run
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Blocked by required conditions
Validate Renovate Config / Validate Renovate Configuration (push) Waiting to run
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
1522 lines
45 KiB
TypeScript
1522 lines
45 KiB
TypeScript
import { describe, expect, it, vi } from 'vitest';
|
|
import { convertResultsToTable } from '../../src/util/convertEvalResultsToTable';
|
|
import { createCompletedPrompt } from '../factories/eval';
|
|
|
|
import type { EvaluateTable, ResultsFile } from '../../src/types/index';
|
|
|
|
describe('convertResultsToTable', () => {
|
|
it('should convert results to table format', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
var1: 'value1',
|
|
var2: 'value2',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
label: 'Test Provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const expected: EvaluateTable = {
|
|
head: {
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['var1', 'var2'],
|
|
},
|
|
body: [
|
|
{
|
|
vars: ['value1', 'value2'],
|
|
outputs: [
|
|
{
|
|
id: 'test1',
|
|
text: 'test output',
|
|
prompt: 'test prompt',
|
|
provider: 'Test Provider',
|
|
pass: true,
|
|
cost: 0,
|
|
success: true,
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
testCase: {},
|
|
promptId: 'prompt1',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
vars: {
|
|
var1: 'value1',
|
|
var2: 'value2',
|
|
},
|
|
audio: undefined,
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
promptIdx: 0,
|
|
testIdx: 0,
|
|
},
|
|
],
|
|
testIdx: 0,
|
|
test: {},
|
|
description: undefined,
|
|
},
|
|
],
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result).toEqual(expected);
|
|
});
|
|
|
|
it('should handle error responses', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
error: 'Test error',
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: false,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
vars: {},
|
|
// @ts-ignore
|
|
failureReason: 'provider_error',
|
|
score: 0,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.body[0].outputs[0].text).toBe('Test error');
|
|
});
|
|
|
|
it('should handle null output by falling back to error', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: null,
|
|
},
|
|
error: 'Provider returned null',
|
|
provider: {
|
|
id: 'test-provider',
|
|
label: 'Test Provider',
|
|
},
|
|
success: false,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'provider_error',
|
|
score: 0,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.body[0].outputs[0].text).toBe('Provider returned null');
|
|
});
|
|
|
|
it('should preserve falsy var values like 0 and false', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
var1: 0,
|
|
var2: false,
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
label: 'Test Provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.body[0].vars).toEqual(['0', 'false']);
|
|
});
|
|
|
|
it('should handle assertion results', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
testCase: {
|
|
assert: [{ type: 'equals', value: 'expected' }],
|
|
},
|
|
gradingResult: {
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'Test failed',
|
|
componentResults: [
|
|
{
|
|
pass: false,
|
|
reason: 'Test failed',
|
|
score: 0,
|
|
},
|
|
],
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: false,
|
|
promptId: 'prompt1',
|
|
vars: {},
|
|
// @ts-ignore
|
|
failureReason: 'assertion_failed',
|
|
score: 0,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.body[0].outputs[0].text).toBe('test output');
|
|
});
|
|
|
|
it('should handle redteam final prompts', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
prompt: 'original prompt',
|
|
},
|
|
metadata: {
|
|
redteamFinalPrompt: 'modified prompt',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.body[0].vars[0]).toBe('modified prompt');
|
|
});
|
|
|
|
it('should handle multiple redteam final prompts', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['prompt', 'query', 'harmCategory'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
prompt: 'original prompt',
|
|
query: 'original query',
|
|
harmCategory: 'test',
|
|
},
|
|
metadata: {
|
|
redteamFinalPrompt: 'modified prompt',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
const vars = result.body[0].vars;
|
|
// Persisted config order should be preserved; see #244, #1320.
|
|
expect(result.head.vars).toEqual(['prompt', 'query', 'harmCategory']);
|
|
expect(vars).toEqual(['modified prompt', 'original query', 'test']);
|
|
});
|
|
|
|
it('should handle audio responses', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
audio: {
|
|
id: 'audio1',
|
|
expiresAt: 1748995200000, // 2025-06-01
|
|
data: 'base64data',
|
|
transcript: 'test transcript',
|
|
format: 'mp3',
|
|
},
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
vars: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.body[0].outputs[0].audio).toEqual({
|
|
id: 'audio1',
|
|
expiresAt: 1748995200000,
|
|
data: 'base64data',
|
|
transcript: 'test transcript',
|
|
format: 'mp3',
|
|
});
|
|
});
|
|
|
|
it('should format object and array variables with pretty-printed JSON', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['conversation', 'simpleVar', 'objectVar'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
conversation: [
|
|
{ user: 'How do I work with you?' },
|
|
{ assistant: 'I can help you understand how to work with me.' },
|
|
{ user: 'Tell me more about NDAs' },
|
|
],
|
|
simpleVar: 'This is a simple string',
|
|
objectVar: { key1: 'value1', key2: 'value2' },
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
label: 'Test Provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
const expectedConversation = JSON.stringify(
|
|
[
|
|
{ user: 'How do I work with you?' },
|
|
{ assistant: 'I can help you understand how to work with me.' },
|
|
{ user: 'Tell me more about NDAs' },
|
|
],
|
|
null,
|
|
2,
|
|
);
|
|
const expectedObject = JSON.stringify({ key1: 'value1', key2: 'value2' }, null, 2);
|
|
|
|
// Persisted config order should be preserved; see #244, #1320.
|
|
expect(result.head.vars).toEqual(['conversation', 'simpleVar', 'objectVar']);
|
|
expect(result.body[0].vars[0]).toBe(expectedConversation);
|
|
expect(result.body[0].vars[1]).toBe('This is a simple string');
|
|
expect(result.body[0].vars[2]).toBe(expectedObject);
|
|
|
|
// Verify that pretty-printed JSON has multiple lines
|
|
expect(result.body[0].vars[0]).toContain('\n');
|
|
expect(result.body[0].vars[2]).toContain('\n');
|
|
});
|
|
|
|
it('should copy sessionId from metadata to vars when vars.sessionId is not present', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionId: 'session-123',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// sessionId should be copied from metadata to vars
|
|
expect(result.head.vars).toContain('sessionId');
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('session-123');
|
|
});
|
|
|
|
it('should not overwrite user-set vars.sessionId with metadata.sessionId', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
sessionId: 'user-session-456',
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionId: 'provider-session-123',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// User-set sessionId should be preserved, not overwritten
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('user-session-456');
|
|
});
|
|
|
|
it('should handle empty vars.sessionId by populating from metadata.sessionId', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
sessionId: '',
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionId: 'session-789',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// Empty sessionId should be populated from metadata
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('session-789');
|
|
});
|
|
|
|
it('should not crash when metadata is missing', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
// Should not throw
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.head.vars).toEqual(['question']);
|
|
});
|
|
|
|
it('should create vars object if missing when populating sessionId from metadata', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
metadata: {
|
|
sessionId: 'session-abc',
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// sessionId should be added even when vars was initially undefined
|
|
expect(result.head.vars).toContain('sessionId');
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('session-abc');
|
|
});
|
|
|
|
it('should handle multiple results with varying sessionId configurations', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
// Result 0: Only metadata.sessionId (should copy to vars)
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 1',
|
|
},
|
|
metadata: {
|
|
sessionId: 'metadata-session-1',
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 1' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
// Result 1: Only vars.sessionId (should preserve)
|
|
{
|
|
id: 'test2',
|
|
testIdx: 1,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 2',
|
|
sessionId: 'user-session-2',
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 2' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
// Result 2: Both metadata and vars sessionId (should preserve vars)
|
|
{
|
|
id: 'test3',
|
|
testIdx: 2,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 3',
|
|
sessionId: 'user-session-3',
|
|
},
|
|
metadata: {
|
|
sessionId: 'metadata-session-3',
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 3' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
// Result 3: Neither metadata nor vars sessionId
|
|
{
|
|
id: 'test4',
|
|
testIdx: 3,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 4',
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 4' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
|
|
// sessionId column should exist (added by results that have it)
|
|
expect(result.head.vars).toContain('sessionId');
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
|
|
// Result 0: metadata.sessionId copied to vars
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('metadata-session-1');
|
|
|
|
// Result 1: user-set vars.sessionId preserved
|
|
expect(result.body[1].vars[sessionIdIndex]).toBe('user-session-2');
|
|
|
|
// Result 2: user-set vars.sessionId preserved (not overwritten by metadata)
|
|
expect(result.body[2].vars[sessionIdIndex]).toBe('user-session-3');
|
|
|
|
// Result 3: no sessionId (empty string in the column)
|
|
expect(result.body[3].vars[sessionIdIndex]).toBe('');
|
|
});
|
|
|
|
it('should copy sessionIds array from metadata to vars for multi-turn strategies', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionIds: ['session-1', 'session-2', 'session-3'],
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// sessionIds array should be joined and copied to vars.sessionId
|
|
expect(result.head.vars).toContain('sessionId');
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('session-1\nsession-2\nsession-3');
|
|
});
|
|
|
|
it('should prefer sessionIds array over sessionId when both exist', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionId: 'single-session',
|
|
sessionIds: ['multi-1', 'multi-2'],
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// sessionIds array should take precedence over sessionId
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('multi-1\nmulti-2');
|
|
});
|
|
|
|
it('should fall back to sessionId when sessionIds is empty array', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionId: 'fallback-session',
|
|
sessionIds: [],
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// Empty sessionIds array should fall back to sessionId
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('fallback-session');
|
|
});
|
|
|
|
it('should handle single-element sessionIds array', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionIds: ['only-session'],
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// Single-element sessionIds should work without trailing comma
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('only-session');
|
|
});
|
|
|
|
it('should ignore non-array sessionIds and fall back to sessionId', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
sessionId: 'fallback-session',
|
|
sessionIds: 'not-an-array' as unknown as string[],
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// Non-array sessionIds should be ignored, falling back to sessionId
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('fallback-session');
|
|
});
|
|
|
|
it('should handle multiple results with varying sessionIds array configurations', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
// Result 0: Has sessionIds array (multi-turn)
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 1',
|
|
},
|
|
metadata: {
|
|
sessionIds: ['multi-1a', 'multi-1b', 'multi-1c'],
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 1' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
// Result 1: Has single sessionId (single-turn)
|
|
{
|
|
id: 'test2',
|
|
testIdx: 1,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 2',
|
|
},
|
|
metadata: {
|
|
sessionId: 'single-session-2',
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 2' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
// Result 2: Has both sessionIds and sessionId (sessionIds takes precedence)
|
|
{
|
|
id: 'test3',
|
|
testIdx: 2,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 3',
|
|
},
|
|
metadata: {
|
|
sessionId: 'ignored-session',
|
|
sessionIds: ['multi-3a', 'multi-3b'],
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 3' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
// Result 3: No sessionId or sessionIds
|
|
{
|
|
id: 'test4',
|
|
testIdx: 3,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'Question 4',
|
|
},
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'output 4' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
|
|
// sessionId column should exist
|
|
expect(result.head.vars).toContain('sessionId');
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
|
|
// Result 0: sessionIds array joined with newlines
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('multi-1a\nmulti-1b\nmulti-1c');
|
|
|
|
// Result 1: single sessionId
|
|
expect(result.body[1].vars[sessionIdIndex]).toBe('single-session-2');
|
|
|
|
// Result 2: sessionIds array takes precedence
|
|
expect(result.body[2].vars[sessionIdIndex]).toBe('multi-3a\nmulti-3b');
|
|
|
|
// Result 3: no sessionId (empty string)
|
|
expect(result.body[3].vars[sessionIdIndex]).toBe('');
|
|
});
|
|
|
|
it('should filter out null, undefined, empty strings, and convert non-strings in sessionIds array', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: {
|
|
question: 'What is AI?',
|
|
},
|
|
metadata: {
|
|
// Array with mixed types: valid strings, null, undefined, empty string, number
|
|
sessionIds: [
|
|
'session-1',
|
|
null,
|
|
undefined,
|
|
'',
|
|
'session-2',
|
|
123,
|
|
'session-3',
|
|
] as unknown as string[],
|
|
},
|
|
prompt: {
|
|
raw: 'test prompt',
|
|
label: 'Test Prompt',
|
|
},
|
|
response: {
|
|
output: 'test output',
|
|
},
|
|
provider: {
|
|
id: 'test-provider',
|
|
},
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
// Should filter out null, undefined, empty strings and convert number to string
|
|
expect(result.head.vars).toContain('sessionId');
|
|
const sessionIdIndex = result.head.vars.indexOf('sessionId');
|
|
expect(result.body[0].vars[sessionIdIndex]).toBe('session-1\nsession-2\n123\nsession-3');
|
|
});
|
|
|
|
it('preserves persisted variable order when loaded results arrive in a different order', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['zebra', 'apple', 'mango', 'banana'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test2',
|
|
testIdx: 1,
|
|
promptIdx: 0,
|
|
vars: { zebra: 'z-row2', apple: 'a-row2', mango: 'm-row2', banana: 'b-row2' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output' },
|
|
provider: { id: 'test-provider', label: 'Test Provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: { zebra: 'z-row1', apple: 'a-row1', mango: 'm-row1' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output' },
|
|
provider: { id: 'test-provider', label: 'Test Provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.head.vars).toEqual(['zebra', 'apple', 'mango', 'banana']);
|
|
expect(result.body[0].vars).toEqual(['z-row1', 'a-row1', 'm-row1', '']);
|
|
expect(result.body[1].vars).toEqual(['z-row2', 'a-row2', 'm-row2', 'b-row2']);
|
|
});
|
|
|
|
it('sorts metadata-only display columns after the persisted variable prefix', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['prompt'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test2',
|
|
testIdx: 1,
|
|
promptIdx: 0,
|
|
vars: { prompt: 'row2' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output', metadata: { transformDisplayVars: { zebra: 'z' } } },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: { prompt: 'row1' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output', metadata: { transformDisplayVars: { apple: 'a' } } },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.head.vars).toEqual(['prompt', 'apple', 'zebra']);
|
|
});
|
|
|
|
it('keeps alphabetical fallback ordering for legacy result files without persisted vars', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: { zebra: 'z', apple: 'a' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
expect(convertResultsToTable(resultsFile).head.vars).toEqual(['apple', 'zebra']);
|
|
});
|
|
|
|
it('appends runtime-only result.vars keys after the persisted prefix in sorted order', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['prompt'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
// `zeta`/`alpha` are NOT in persistedVars and NOT transformDisplayVars.
|
|
// They should be appended after `prompt` in alphabetical order.
|
|
vars: { prompt: 'p1', zeta: 'z1', alpha: 'a1' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.head.vars).toEqual(['prompt', 'alpha', 'zeta']);
|
|
expect(result.body[0].vars).toEqual(['p1', 'a1', 'z1']);
|
|
});
|
|
|
|
it.each([
|
|
['empty string', '', ''],
|
|
['zero', 0, '0'],
|
|
['false', false, 'false'],
|
|
])('does not overwrite a falsy result.vars value (%s) with transformDisplayVars', (_label, falsyValue, expectedRendered) => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['prompt'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
// Falsy value for `prompt` must be preserved, not silently
|
|
// replaced by transformDisplayVars.prompt.
|
|
vars: { prompt: falsyValue } as unknown as Record<string, string>,
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: {
|
|
output: 'test output',
|
|
metadata: { transformDisplayVars: { prompt: 'from-transform' } },
|
|
},
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.head.vars).toEqual(['prompt']);
|
|
expect(result.body[0].vars).toEqual([expectedRendered]);
|
|
});
|
|
|
|
it('treats a malformed non-array vars field as if no order were persisted', () => {
|
|
// Bogus payload from a corrupt store / older writer. Cast via unknown
|
|
// because the malformed shape intentionally does not match the type.
|
|
const resultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: 'not-an-array',
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: { zebra: 'z', apple: 'a' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
} as unknown as ResultsFile;
|
|
|
|
expect(convertResultsToTable(resultsFile).head.vars).toEqual(['apple', 'zebra']);
|
|
});
|
|
|
|
it('logs a debug breadcrumb when transformDisplayVars collides with a different result.vars value', async () => {
|
|
const logger = (await import('../../src/logger')).default;
|
|
const debugSpy = vi.spyOn(logger, 'debug').mockImplementation(() => logger);
|
|
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['prompt'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: { prompt: 'original' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: {
|
|
output: 'test output',
|
|
metadata: { transformDisplayVars: { prompt: 'shadowed' } },
|
|
},
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
|
|
expect(result.body[0].vars).toEqual(['original']);
|
|
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining("key 'prompt' collides"));
|
|
debugSpy.mockRestore();
|
|
});
|
|
|
|
it('dedups a corrupt persisted vars list so duplicate header columns are not rendered', () => {
|
|
const resultsFile: ResultsFile = {
|
|
version: 4,
|
|
prompts: [createCompletedPrompt('test prompt', { display: 'test prompt', id: 'prompt1' })],
|
|
vars: ['prompt', 'prompt', 'foo'],
|
|
results: {
|
|
results: [
|
|
{
|
|
id: 'test1',
|
|
testIdx: 0,
|
|
promptIdx: 0,
|
|
vars: { prompt: 'p', foo: 'f' },
|
|
prompt: { raw: 'test prompt', label: 'Test Prompt' },
|
|
response: { output: 'test output' },
|
|
provider: { id: 'test-provider' },
|
|
success: true,
|
|
promptId: 'prompt1',
|
|
testCase: {},
|
|
// @ts-ignore
|
|
failureReason: 'none',
|
|
score: 1,
|
|
latencyMs: 100,
|
|
namedScores: {},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
const result = convertResultsToTable(resultsFile);
|
|
expect(result.head.vars).toEqual(['prompt', 'foo']);
|
|
expect(result.body[0].vars).toEqual(['p', 'f']);
|
|
});
|
|
});
|