chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,419 @@
import { describe, expect, it } from 'vitest';
import { convertTestResultsToTableRow } from '../../../src/util/exportToFile/index';
import type EvalResult from '../../../src/models/evalResult';
/**
* Creates a minimal mock EvalResult for testing.
* We use a partial type and cast to avoid needing the full EvalResult class.
*/
function createMockEvalResult(overrides: Partial<EvalResult> = {}): EvalResult {
const defaultResult = {
id: 'test-id',
testIdx: 0,
promptIdx: 0,
description: 'Test description',
testCase: {
vars: {},
},
prompt: {
raw: 'test prompt',
},
response: {
output: 'test output',
},
provider: {
id: 'test-provider',
label: 'Test Provider',
},
success: true,
score: 1,
latencyMs: 100,
error: undefined,
metadata: undefined,
namedScores: {},
gradingResult: undefined,
cost: 0,
};
return {
...defaultResult,
...overrides,
testCase: {
...defaultResult.testCase,
...(overrides.testCase || {}),
},
prompt: {
...defaultResult.prompt,
...(overrides.prompt || {}),
},
response: {
...defaultResult.response,
...(overrides.response || {}),
},
provider: {
...defaultResult.provider,
...(overrides.provider || {}),
},
} as EvalResult;
}
describe('convertTestResultsToTableRow', () => {
describe('basic functionality', () => {
it('should convert results to table row format', () => {
const results = [
createMockEvalResult({
testCase: { vars: { question: 'What is AI?' } },
}),
];
const varsForHeader = ['question'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.description).toBe('Test description');
expect(row.vars).toEqual(['What is AI?']);
expect(row.testIdx).toBe(0);
expect(row.outputs).toHaveLength(1);
expect(row.outputs[0].text).toBe('test output');
});
it('should handle multiple variables', () => {
const results = [
createMockEvalResult({
testCase: {
vars: {
question: 'What is AI?',
context: 'Some context',
temperature: 0.5,
},
},
}),
];
const varsForHeader = ['context', 'question', 'temperature'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['Some context', 'What is AI?', '0.5']);
});
it('should handle missing variables', () => {
const results = [
createMockEvalResult({
testCase: { vars: { question: 'What is AI?' } },
}),
];
const varsForHeader = ['question', 'missingVar'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['What is AI?', '']);
});
it('should handle object variables as JSON', () => {
const results = [
createMockEvalResult({
testCase: {
vars: {
config: { nested: 'value', count: 42 },
},
},
}),
];
const varsForHeader = ['config'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars[0]).toBe('{"nested":"value","count":42}');
});
});
describe('sessionId handling', () => {
it('should use sessionId from testCase.vars when present', () => {
const results = [
createMockEvalResult({
testCase: { vars: { sessionId: 'user-session-123' } },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['user-session-123']);
});
it('should use metadata.sessionId when vars.sessionId is not present', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: { sessionId: 'metadata-session-456' },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['metadata-session-456']);
});
it('should use metadata.sessionId when vars.sessionId is empty string', () => {
const results = [
createMockEvalResult({
testCase: { vars: { sessionId: '' } },
metadata: { sessionId: 'metadata-session-789' },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['metadata-session-789']);
});
it('should not overwrite non-empty vars.sessionId with metadata.sessionId', () => {
const results = [
createMockEvalResult({
testCase: { vars: { sessionId: 'user-provided-session' } },
metadata: { sessionId: 'should-be-ignored' },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['user-provided-session']);
});
it('should return empty string when no sessionId is available', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: {},
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['']);
});
});
describe('sessionIds array handling (multi-turn strategies)', () => {
it('should join sessionIds array with newlines', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: { sessionIds: ['session-1', 'session-2', 'session-3'] },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['session-1\nsession-2\nsession-3']);
});
it('should handle single-element sessionIds array', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: { sessionIds: ['only-session'] },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['only-session']);
});
it('should prefer sessionIds array over sessionId when both exist', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: {
sessionId: 'single-session',
sessionIds: ['multi-1', 'multi-2'],
},
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['multi-1\nmulti-2']);
});
it('should fall back to sessionId when sessionIds is empty array', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: {
sessionId: 'fallback-session',
sessionIds: [],
},
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['fallback-session']);
});
it('should ignore non-array sessionIds and use sessionId instead', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: {
sessionId: 'fallback-session',
sessionIds: 'not-an-array' as unknown as string[],
},
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['fallback-session']);
});
it('should not use sessionIds array if vars.sessionId is provided', () => {
const results = [
createMockEvalResult({
testCase: { vars: { sessionId: 'user-provided' } },
metadata: { sessionIds: ['should-be-ignored-1', 'should-be-ignored-2'] },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['user-provided']);
});
});
describe('combined scenarios', () => {
it('should handle sessionId with other variables', () => {
const results = [
createMockEvalResult({
testCase: {
vars: {
question: 'What is AI?',
context: 'Some context',
},
},
metadata: { sessionIds: ['session-a', 'session-b'] },
}),
];
const varsForHeader = ['context', 'question', 'sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['Some context', 'What is AI?', 'session-a\nsession-b']);
});
it('should handle edge case with undefined metadata', () => {
const results = [
createMockEvalResult({
testCase: { vars: { question: 'What is AI?' } },
metadata: undefined,
}),
];
const varsForHeader = ['question', 'sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.vars).toEqual(['What is AI?', '']);
});
it('should handle non-string sessionId values in metadata', () => {
const results = [
createMockEvalResult({
testCase: { vars: {} },
metadata: { sessionId: { complex: 'object' } as unknown as string },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
// Should be JSON stringified
expect(row.vars[0]).toBe('{"complex":"object"}');
});
it('should handle non-string sessionId values in vars', () => {
const results = [
createMockEvalResult({
testCase: { vars: { sessionId: { complex: 'object' } as unknown as string } },
}),
];
const varsForHeader = ['sessionId'];
const row = convertTestResultsToTableRow(results, varsForHeader);
// Should be JSON stringified
expect(row.vars[0]).toBe('{"complex":"object"}');
});
});
describe('output formatting', () => {
it('should format successful output correctly', () => {
const results = [
createMockEvalResult({
success: true,
response: { output: 'The answer is 42' },
}),
];
const varsForHeader: string[] = [];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.outputs[0].text).toBe('The answer is 42');
expect(row.outputs[0].pass).toBe(true);
});
it('should format error output correctly', () => {
const results = [
createMockEvalResult({
success: false,
error: 'Provider error occurred',
response: { output: '' },
}),
];
const varsForHeader: string[] = [];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.outputs[0].text).toBe('Provider error occurred');
expect(row.outputs[0].pass).toBe(false);
});
it('should handle null output', () => {
const results = [
createMockEvalResult({
response: { output: null },
error: 'Null response',
}),
];
const varsForHeader: string[] = [];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.outputs[0].text).toBe('Null response');
});
it('should handle object output', () => {
const results = [
createMockEvalResult({
response: { output: { key: 'value', nested: { data: 123 } } },
}),
];
const varsForHeader: string[] = [];
const row = convertTestResultsToTableRow(results, varsForHeader);
expect(row.outputs[0].text).toBe('{"key":"value","nested":{"data":123}}');
});
});
});
@@ -0,0 +1,178 @@
import { beforeEach, describe, expect, it } from 'vitest';
import { getHeaderForTable } from '../../../src/util/exportToFile/getHeaderForTable';
import type Eval from '../../../src/models/eval';
import type { TestCase } from '../../../src/types/index';
describe('getHeaderForTable', () => {
let mockEval: Eval;
beforeEach(() => {
mockEval = {
config: {
defaultTest: undefined,
},
results: [],
} as any;
});
it('should handle object defaultTest with vars', () => {
mockEval.config.defaultTest = {
vars: {
defaultVar1: 'value1',
defaultVar2: 'value2',
},
};
mockEval.results = [
{
testCase: {
vars: {
testVar1: 'test1',
defaultVar1: 'override1',
},
} as TestCase,
},
{
testCase: {
vars: {
testVar2: 'test2',
testVar3: 'test3',
},
} as TestCase,
},
] as any;
const header = getHeaderForTable(mockEval);
// Should include all unique var names from defaultTest and test cases
expect(header.vars).toContain('defaultVar1');
expect(header.vars).toContain('defaultVar2');
expect(header.vars).toContain('testVar1');
expect(header.vars).toContain('testVar2');
expect(header.vars).toContain('testVar3');
});
it('should handle string defaultTest', () => {
mockEval.config.defaultTest = 'file://path/to/defaultTest.yaml';
mockEval.results = [
{
testCase: {
vars: {
var1: 'value1',
var2: 'value2',
},
} as TestCase,
},
] as any;
const header = getHeaderForTable(mockEval);
// Should only include vars from test cases when defaultTest is a string
expect(header.vars).toContain('var1');
expect(header.vars).toContain('var2');
});
it('should handle undefined defaultTest', () => {
mockEval.config.defaultTest = undefined;
mockEval.results = [
{
testCase: {
vars: {
var1: 'value1',
var2: 'value2',
},
} as TestCase,
},
] as any;
const header = getHeaderForTable(mockEval);
expect(header.vars).toContain('var1');
expect(header.vars).toContain('var2');
});
it('should handle empty vars in defaultTest', () => {
mockEval.config.defaultTest = {
vars: {},
assert: [{ type: 'equals', value: 'test' }],
};
mockEval.results = [
{
testCase: {
vars: {
var1: 'value1',
},
} as TestCase,
},
] as any;
const header = getHeaderForTable(mockEval);
expect(header.vars).toContain('var1');
});
it('should handle defaultTest with no vars property', () => {
mockEval.config.defaultTest = {
assert: [{ type: 'equals', value: 'test' }],
options: { provider: 'openai:gpt-4' },
};
mockEval.results = [
{
testCase: {
vars: {
var1: 'value1',
},
} as TestCase,
},
] as any;
const header = getHeaderForTable(mockEval);
expect(header.vars).toContain('var1');
});
it('should deduplicate var names', () => {
mockEval.config.defaultTest = {
vars: {
commonVar: 'default',
defaultOnly: 'value',
},
};
mockEval.results = [
{
testCase: {
vars: {
commonVar: 'override1',
test1Var: 'value1',
},
} as TestCase,
},
{
testCase: {
vars: {
commonVar: 'override2',
test2Var: 'value2',
},
} as TestCase,
},
] as any;
const header = getHeaderForTable(mockEval);
// Count occurrences of commonVar
const commonVarCount = header.vars.filter((name) => name === 'commonVar').length;
expect(commonVarCount).toBe(1);
// All vars should be present
expect(header.vars).toContain('commonVar');
expect(header.vars).toContain('defaultOnly');
expect(header.vars).toContain('test1Var');
expect(header.vars).toContain('test2Var');
});
});
+382
View File
@@ -0,0 +1,382 @@
import { describe, expect, it } from 'vitest';
import { ResultFailureReason } from '../../../src/types/index';
import { getHeaderForTable } from '../../../src/util/exportToFile/getHeaderForTable';
import {
convertEvalResultToTableCell,
convertTestResultsToTableRow,
} from '../../../src/util/exportToFile/index';
import type Eval from '../../../src/models/eval';
import type EvalResult from '../../../src/models/evalResult';
describe('exportToFile utils', () => {
describe('getHeaderForTable', () => {
it('should extract vars from defaultTest', () => {
const eval_: Partial<Eval> = {
id: 'test-id',
createdAt: Date.now(),
config: {
defaultTest: {
vars: {
var1: 'value1',
var2: 'value2',
},
},
},
prompts: [],
results: [],
persisted: false,
};
const result = getHeaderForTable(eval_ as Eval);
expect(result.vars).toEqual(['var1', 'var2']);
});
it('should extract vars from tests array', () => {
const eval_: Partial<Eval> = {
id: 'test-id',
createdAt: Date.now(),
config: {
tests: [
{
vars: {
var3: 'value3',
var4: 'value4',
},
},
],
},
prompts: [],
results: [],
persisted: false,
};
const result = getHeaderForTable(eval_ as Eval);
expect(result.vars).toEqual(['var3', 'var4']);
});
it('should extract vars from scenarios', () => {
const eval_: Partial<Eval> = {
id: 'test-id',
createdAt: Date.now(),
config: {
scenarios: [
{
config: [
{
vars: {
var5: 'value5',
},
},
],
tests: [
{
vars: {
var6: 'value6',
},
},
],
},
],
},
prompts: [],
results: [],
persisted: false,
};
const result = getHeaderForTable(eval_ as Eval);
expect(result.vars).toEqual(['var5', 'var6']);
});
it('should handle empty config', () => {
const eval_: Partial<Eval> = {
id: 'test-id',
createdAt: Date.now(),
config: {},
prompts: [],
results: [],
persisted: false,
};
const result = getHeaderForTable(eval_ as Eval);
expect(result.vars).toEqual([]);
});
});
describe('convertEvalResultToTableCell', () => {
it('should handle successful assertion result', () => {
const result: Partial<EvalResult> = {
id: 'test-1',
evalId: 'eval-1',
testCase: {
assert: [
{
type: 'contains',
value: 'test',
},
],
},
success: true,
response: {
output: 'test output',
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
};
const output = convertEvalResultToTableCell(result as EvalResult);
expect(output.text).toBe('test output');
expect(output.pass).toBe(true);
});
it('should handle failed assertion result', () => {
const result: Partial<EvalResult> = {
id: 'test-1',
evalId: 'eval-1',
testCase: {
assert: [
{
type: 'contains',
value: 'test',
},
],
},
success: false,
error: 'test error',
response: {
output: 'test output',
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
gradingResult: {
pass: false,
score: 0,
reason: 'test failure',
componentResults: [
{
pass: false,
score: 0,
reason: 'test failure',
},
],
},
failureReason: ResultFailureReason.ASSERT,
};
const output = convertEvalResultToTableCell(result as EvalResult);
expect(output.text).toBe('test output');
expect(output.pass).toBe(false);
});
it('should handle error without assertion', () => {
const result: Partial<EvalResult> = {
id: 'test-1',
evalId: 'eval-1',
testCase: {},
error: 'test error',
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.ERROR,
};
const output = convertEvalResultToTableCell(result as EvalResult);
expect(output.text).toBe('test error');
});
it('should preserve falsy outputs like 0 and false', () => {
const numericResult: Partial<EvalResult> = {
id: 'test-1',
evalId: 'eval-1',
testCase: {},
response: {
output: 0,
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
};
const booleanResult: Partial<EvalResult> = {
id: 'test-2',
evalId: 'eval-1',
testCase: {},
response: {
output: false,
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
};
const numericOutput = convertEvalResultToTableCell(numericResult as EvalResult);
const booleanOutput = convertEvalResultToTableCell(booleanResult as EvalResult);
expect(numericOutput.text).toBe('0');
expect(booleanOutput.text).toBe('false');
});
it('should handle null output by falling back to error', () => {
const resultWithError: Partial<EvalResult> = {
id: 'test-1',
evalId: 'eval-1',
testCase: {},
response: {
output: null,
},
error: 'Provider returned null',
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.ERROR,
};
const resultWithoutError: Partial<EvalResult> = {
id: 'test-2',
evalId: 'eval-1',
testCase: {},
response: {
output: null,
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
};
const outputWithError = convertEvalResultToTableCell(resultWithError as EvalResult);
const outputWithoutError = convertEvalResultToTableCell(resultWithoutError as EvalResult);
expect(outputWithError.text).toBe('Provider returned null');
expect(outputWithoutError.text).toBe('');
});
});
describe('convertTestResultsToTableRow', () => {
it('should convert results to table row', () => {
const results: Partial<EvalResult>[] = [
{
id: 'test-1',
evalId: 'eval-1',
description: 'test description',
promptIdx: 0,
testCase: {
vars: {
var1: 'value1',
var2: 'value2',
},
},
response: {
output: 'test output',
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
},
];
const varsForHeader = ['var1', 'var2'];
const row = convertTestResultsToTableRow(results as EvalResult[], varsForHeader);
expect(row.description).toBe('test description');
expect(row.vars).toEqual(['value1', 'value2']);
expect(row.outputs[0].text).toBe('test output');
});
it('should handle complex var values', () => {
const results: Partial<EvalResult>[] = [
{
id: 'test-1',
evalId: 'eval-1',
promptIdx: 0,
testCase: {
vars: {
var1: { complex: 'object' },
},
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
},
];
const varsForHeader = ['var1'];
const row = convertTestResultsToTableRow(results as EvalResult[], varsForHeader);
expect(row.vars).toEqual(['{"complex":"object"}']);
});
it('should preserve falsy var values like 0 and false', () => {
const results: Partial<EvalResult>[] = [
{
id: 'test-1',
evalId: 'eval-1',
promptIdx: 0,
testCase: {
vars: {
var1: 0,
var2: false,
},
},
prompt: {
raw: 'test prompt',
label: 'test',
},
provider: {
id: 'test-provider',
},
failureReason: ResultFailureReason.NONE,
},
];
const varsForHeader = ['var1', 'var2'];
const row = convertTestResultsToTableRow(results as EvalResult[], varsForHeader);
expect(row.vars).toEqual(['0', 'false']);
});
});
});