wehub-resource-sync
0d3cb498a3
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Waiting to run
Test and Publish Multi-arch Docker Image / test (push) Waiting to run
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Blocked by required conditions
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Blocked by required conditions
Validate Renovate Config / Validate Renovate Configuration (push) Waiting to run
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
4006 lines
120 KiB
TypeScript
4006 lines
120 KiB
TypeScript
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { runAssertion } from '../../src/assertions/index';
|
|
import { OpenAiChatCompletionProvider } from '../../src/providers/openai/chat';
|
|
import { DefaultEmbeddingProvider } from '../../src/providers/openai/defaults';
|
|
import { fetchWithRetries } from '../../src/util/fetch/index';
|
|
import { createMockProvider } from '../factories/provider';
|
|
import { TestGrader } from '../util/utils';
|
|
|
|
import type { ApiProvider, Assertion, AtomicTestCase, GradingResult } from '../../src/types/index';
|
|
|
|
vi.mock('../../src/redteam/remoteGeneration', () => ({
|
|
shouldGenerateRemote: vi.fn().mockReturnValue(false),
|
|
}));
|
|
|
|
// Causes a SIGSEGV in github actions.
|
|
vi.mock('libsql');
|
|
|
|
vi.mock('proxy-agent', () => ({
|
|
ProxyAgent: vi.fn().mockImplementation(() => ({})),
|
|
}));
|
|
|
|
vi.mock('node:module', () => {
|
|
const mockRequire: NodeJS.Require = {
|
|
resolve: vi.fn() as unknown as NodeJS.RequireResolve,
|
|
} as unknown as NodeJS.Require;
|
|
return {
|
|
createRequire: vi.fn().mockReturnValue(mockRequire),
|
|
};
|
|
});
|
|
|
|
vi.mock('../../src/util/fetch/index.ts', async () => {
|
|
const actual = await vi.importActual<typeof import('../../src/util/fetch/index')>(
|
|
'../../src/util/fetch/index.ts',
|
|
);
|
|
return {
|
|
...actual,
|
|
fetchWithRetries: vi.fn(actual.fetchWithRetries),
|
|
};
|
|
});
|
|
|
|
vi.mock('glob', () => ({
|
|
globSync: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('fs', () => {
|
|
const fsMock = {
|
|
readFileSync: vi.fn(),
|
|
existsSync: vi.fn(),
|
|
writeFileSync: vi.fn(),
|
|
mkdirSync: vi.fn(),
|
|
promises: {
|
|
readFile: vi.fn(),
|
|
},
|
|
};
|
|
return {
|
|
...fsMock,
|
|
default: fsMock,
|
|
};
|
|
});
|
|
|
|
vi.mock('../../src/esm', () => ({
|
|
getDirectory: () => '/test/dir',
|
|
importModule: vi.fn((_filePath: string, functionName?: string) => {
|
|
return Promise.resolve(functionName ? {} : undefined);
|
|
}),
|
|
}));
|
|
vi.mock('../../src/database', () => ({
|
|
getDb: vi.fn(),
|
|
}));
|
|
vi.mock('path', async () => {
|
|
const actual = await vi.importActual<typeof import('path')>('path');
|
|
const mocked = {
|
|
...actual,
|
|
resolve: vi.fn(),
|
|
extname: vi.fn(),
|
|
};
|
|
return {
|
|
...mocked,
|
|
default: mocked,
|
|
};
|
|
});
|
|
|
|
vi.mock('../../src/cliState', () => ({
|
|
default: {
|
|
basePath: '/base/path',
|
|
},
|
|
basePath: '/base/path',
|
|
}));
|
|
vi.mock('../../src/matchers/rag', async () => {
|
|
const actual =
|
|
await vi.importActual<typeof import('../../src/matchers/rag')>('../../src/matchers/rag');
|
|
return {
|
|
...actual,
|
|
matchesContextRelevance: vi
|
|
.fn()
|
|
.mockResolvedValue({ pass: true, score: 1, reason: 'Mocked reason' }),
|
|
matchesContextFaithfulness: vi
|
|
.fn()
|
|
.mockResolvedValue({ pass: true, score: 1, reason: 'Mocked reason' }),
|
|
};
|
|
});
|
|
|
|
const Grader = new TestGrader();
|
|
|
|
describe('runAssertion', () => {
|
|
afterEach(() => {
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
const equalityAssertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'Expected output',
|
|
};
|
|
|
|
const equalityAssertionWithObject: Assertion = {
|
|
type: 'equals',
|
|
value: { key: 'value' },
|
|
};
|
|
|
|
const isJsonAssertion: Assertion = {
|
|
type: 'is-json',
|
|
};
|
|
|
|
const isJsonAssertionWithSchema: Assertion = {
|
|
type: 'is-json',
|
|
value: {
|
|
required: ['latitude', 'longitude'],
|
|
type: 'object',
|
|
properties: {
|
|
latitude: {
|
|
type: 'number',
|
|
minimum: -90,
|
|
maximum: 90,
|
|
},
|
|
longitude: {
|
|
type: 'number',
|
|
minimum: -180,
|
|
maximum: 180,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
const isJsonAssertionWithSchemaYamlString: Assertion = {
|
|
type: 'is-json',
|
|
value: `
|
|
required: ["latitude", "longitude"]
|
|
type: object
|
|
properties:
|
|
latitude:
|
|
type: number
|
|
minimum: -90
|
|
maximum: 90
|
|
longitude:
|
|
type: number
|
|
minimum: -180
|
|
maximum: 180
|
|
`,
|
|
};
|
|
|
|
const isSqlAssertion: Assertion = {
|
|
type: 'is-sql',
|
|
};
|
|
|
|
const notIsSqlAssertion: Assertion = {
|
|
type: 'not-is-sql',
|
|
};
|
|
|
|
const isSqlAssertionWithDatabase: Assertion = {
|
|
type: 'is-sql',
|
|
value: {
|
|
databaseType: 'MySQL',
|
|
},
|
|
};
|
|
|
|
const isSqlAssertionWithDatabaseAndWhiteTableList: Assertion = {
|
|
type: 'is-sql',
|
|
value: {
|
|
databaseType: 'MySQL',
|
|
allowedTables: ['(select|update|insert|delete)::null::departments'],
|
|
},
|
|
};
|
|
|
|
const isSqlAssertionWithDatabaseAndWhiteColumnList: Assertion = {
|
|
type: 'is-sql',
|
|
value: {
|
|
databaseType: 'MySQL',
|
|
allowedColumns: ['select::null::name', 'update::null::id'],
|
|
},
|
|
};
|
|
|
|
const isSqlAssertionWithDatabaseAndBothList: Assertion = {
|
|
type: 'is-sql',
|
|
value: {
|
|
databaseType: 'MySQL',
|
|
allowedTables: ['(select|update|insert|delete)::null::departments'],
|
|
allowedColumns: ['select::null::name', 'update::null::id'],
|
|
},
|
|
};
|
|
|
|
const containsJsonAssertion: Assertion = {
|
|
type: 'contains-json',
|
|
};
|
|
|
|
const containsJsonAssertionWithSchema: Assertion = {
|
|
type: 'contains-json',
|
|
value: {
|
|
required: ['latitude', 'longitude'],
|
|
type: 'object',
|
|
properties: {
|
|
latitude: {
|
|
type: 'number',
|
|
minimum: -90,
|
|
maximum: 90,
|
|
},
|
|
longitude: {
|
|
type: 'number',
|
|
minimum: -180,
|
|
maximum: 180,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
const javascriptStringAssertion: Assertion = {
|
|
type: 'javascript',
|
|
value: 'output === "Expected output"',
|
|
};
|
|
|
|
const javascriptStringAssertionWithNumber: Assertion = {
|
|
type: 'javascript',
|
|
value: 'output.length * 10',
|
|
};
|
|
|
|
const javascriptBooleanAssertionWithConfig: Assertion = {
|
|
type: 'javascript',
|
|
value: 'output.length <= context.config.maximumOutputSize',
|
|
config: {
|
|
maximumOutputSize: 20,
|
|
},
|
|
};
|
|
|
|
const javascriptStringAssertionWithNumberAndThreshold: Assertion = {
|
|
type: 'javascript',
|
|
value: 'output.length * 10',
|
|
threshold: 0.5,
|
|
};
|
|
|
|
it('should pass when the equality assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: equalityAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the equality assertion fails', async () => {
|
|
const output = 'Actual output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: equalityAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output "Actual output" to equal "Expected output"',
|
|
});
|
|
});
|
|
|
|
const notEqualsAssertion: Assertion = {
|
|
type: 'not-equals',
|
|
value: 'Unexpected output',
|
|
};
|
|
|
|
it('should pass when the not-equals assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notEqualsAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the not-equals assertion fails', async () => {
|
|
const output = 'Unexpected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notEqualsAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output "Unexpected output" to not equal "Unexpected output"',
|
|
});
|
|
});
|
|
|
|
it('should handle output as an object', async () => {
|
|
const output = { key: 'value' };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: equalityAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output "{"key":"value"}" to equal "Expected output"',
|
|
});
|
|
});
|
|
|
|
it('should pass when the equality assertion with object passes', async () => {
|
|
const output = { key: 'value' };
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: equalityAssertionWithObject,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the equality assertion with object fails', async () => {
|
|
const output = { key: 'not value' };
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: equalityAssertionWithObject,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output "{"key":"not value"}" to equal "{"key":"value"}"',
|
|
});
|
|
});
|
|
|
|
it('should pass when the equality assertion with object passes with external json', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'file:///output.json',
|
|
};
|
|
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/output.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ key: 'value' }));
|
|
|
|
const output = '{"key":"value"}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/output.json', 'utf8');
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the equality assertion with object fails with external object', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'file:///output.json',
|
|
};
|
|
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/output.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ key: 'value' }));
|
|
|
|
const output = '{"key":"not value"}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/output.json', 'utf8');
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output "{"key":"not value"}" to equal "{"key":"value"}"',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-json assertion passes', async () => {
|
|
const output = '{"key":"value"}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isJsonAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-json assertion fails', async () => {
|
|
const output = 'Not valid JSON';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isJsonAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to be valid JSON',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-json assertion passes with schema', async () => {
|
|
const output = '{"latitude": 80.123, "longitude": -1}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isJsonAssertionWithSchema,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-json assertion fails with schema', async () => {
|
|
const output = '{"latitude": "high", "longitude": [-1]}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isJsonAssertionWithSchema,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-json assertion passes with schema YAML string', async () => {
|
|
const output = '{"latitude": 80.123, "longitude": -1}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isJsonAssertionWithSchemaYamlString,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-json assertion fails with schema YAML string', async () => {
|
|
const output = '{"latitude": "high", "longitude": [-1]}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isJsonAssertionWithSchemaYamlString,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
|
|
});
|
|
});
|
|
|
|
it('should fail when the not-is-json assertion finds matching schema', async () => {
|
|
const output = '{"latitude": 80.123, "longitude": -1}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'not-is-json',
|
|
value: isJsonAssertionWithSchema.value,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'Output is JSON that conforms to the provided schema',
|
|
});
|
|
});
|
|
|
|
it('should pass when the not-is-json assertion finds non-matching schema', async () => {
|
|
const output = '{"latitude": "high", "longitude": [-1]}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'not-is-json',
|
|
value: isJsonAssertionWithSchema.value,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the not-is-json assertion with schema receives non-JSON input', async () => {
|
|
const output = 'this is not json at all';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'not-is-json',
|
|
value: isJsonAssertionWithSchema.value,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should validate JSON with formats using ajv-formats', async () => {
|
|
const output = '{"date": "2021-08-29"}';
|
|
const schemaWithFormat = {
|
|
type: 'object',
|
|
properties: {
|
|
date: {
|
|
type: 'string',
|
|
format: 'date',
|
|
},
|
|
},
|
|
required: ['date'],
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: { type: 'is-json', value: schemaWithFormat },
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should validate JSON with formats using ajv-formats - failure', async () => {
|
|
const output = '{"date": "not a date"}';
|
|
const schemaWithFormat = {
|
|
type: 'object',
|
|
properties: {
|
|
date: {
|
|
type: 'string',
|
|
format: 'date',
|
|
},
|
|
},
|
|
required: ['date'],
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: { type: 'is-json', value: schemaWithFormat },
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason:
|
|
'JSON does not conform to the provided schema. Errors: data/date must match format "date"',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-json assertion passes with external schema', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'is-json',
|
|
value: 'file:///schema.json',
|
|
};
|
|
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
vi.mocked(fs.readFileSync).mockReturnValue(
|
|
JSON.stringify({
|
|
required: ['latitude', 'longitude'],
|
|
type: 'object',
|
|
properties: {
|
|
latitude: {
|
|
type: 'number',
|
|
minimum: -90,
|
|
maximum: 90,
|
|
},
|
|
longitude: {
|
|
type: 'number',
|
|
minimum: -180,
|
|
maximum: 180,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
|
|
const output = '{"latitude": 80.123, "longitude": -1}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-json assertion fails with external schema', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'is-json',
|
|
value: 'file:///schema.json',
|
|
};
|
|
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
vi.mocked(fs.readFileSync).mockReturnValue(
|
|
JSON.stringify({
|
|
required: ['latitude', 'longitude'],
|
|
type: 'object',
|
|
properties: {
|
|
latitude: {
|
|
type: 'number',
|
|
minimum: -90,
|
|
maximum: 90,
|
|
},
|
|
longitude: {
|
|
type: 'number',
|
|
minimum: -180,
|
|
maximum: 180,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
|
|
const output = '{"latitude": "high", "longitude": [-1]}';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-sql assertion passes', async () => {
|
|
const output = 'SELECT id, name FROM users';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails', async () => {
|
|
const output = 'SELECT * FROM orders ORDERY BY order_date';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'SQL statement does not conform to the provided MySQL database syntax.',
|
|
});
|
|
});
|
|
|
|
it('should pass when the not-is-sql assertion passes', async () => {
|
|
const output = 'SELECT * FROM orders ORDERY BY order_date';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: notIsSqlAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the not-is-sql assertion fails', async () => {
|
|
const output = 'SELECT id, name FROM users';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: notIsSqlAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'The output SQL statement is valid',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-sql assertion passes given MySQL Database syntax', async () => {
|
|
const output = 'SELECT id, name FROM users';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabase,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails given MySQL Database syntax', async () => {
|
|
const output = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabase,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'SQL statement does not conform to the provided MySQL database syntax.',
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-sql assertion passes given MySQL Database syntax and allowedTables', async () => {
|
|
const output = 'SELECT * FROM departments WHERE department_id = 1';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails given MySQL Database syntax and allowedTables', async () => {
|
|
const output = 'UPDATE employees SET department_id = 2 WHERE employee_id = 1';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: `SQL references unauthorized table(s). Found: [update::null::employees]. Allowed: [(select|update|insert|delete)::null::departments].`,
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-sql assertion passes given MySQL Database syntax and allowedColumns', async () => {
|
|
const output = 'SELECT name FROM t';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails given MySQL Database syntax and allowedColumns', async () => {
|
|
const output = 'SELECT age FROM a WHERE id = 1';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: `SQL references unauthorized column(s). Found: [select::null::age, select::null::id]. Allowed: [select::null::name, update::null::id].`,
|
|
});
|
|
});
|
|
|
|
it('should pass when the is-sql assertion passes given MySQL Database syntax, allowedTables, and allowedColumns', async () => {
|
|
const output = 'SELECT name FROM departments';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndBothList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails given MySQL Database syntax, allowedTables, and allowedColumns', async () => {
|
|
const output = `INSERT INTO departments (name) VALUES ('HR')`;
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndBothList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: `SQL references unauthorized column(s). Found: [insert::departments::name]. Allowed: [select::null::name, update::null::id].`,
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails due to missing table authority for MySQL Database syntax', async () => {
|
|
const output = 'UPDATE a SET id = 1';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndBothList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: `SQL references unauthorized table(s). Found: [update::null::a]. Allowed: [(select|update|insert|delete)::null::departments].`,
|
|
});
|
|
});
|
|
|
|
it('should fail when the is-sql assertion fails due to missing authorities for DELETE statement in MySQL Database syntax', async () => {
|
|
const output = `DELETE FROM employees;`;
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: isSqlAssertionWithDatabaseAndBothList,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: `SQL references unauthorized table(s). Found: [delete::null::employees]. Allowed: [(select|update|insert|delete)::null::departments]. SQL references unauthorized column(s). Found: [delete::employees::(.*)]. Allowed: [select::null::name, update::null::id].`,
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-sql assertion passes', async () => {
|
|
const output = 'wassup\n```\nSELECT id, name FROM users\n```\nyolo';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'contains-sql',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-sql assertion sees `sql` in code block', async () => {
|
|
const output = 'wassup\n```sql\nSELECT id, name FROM users\n```\nyolo';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'contains-sql',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-sql assertion sees sql without code block', async () => {
|
|
const output = 'SELECT id, name FROM users';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'contains-sql',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the contains-sql does not contain code block', async () => {
|
|
const output = 'nothin';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'contains-sql',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
});
|
|
});
|
|
|
|
it('should fail when the contains-sql does not contain sql in code block', async () => {
|
|
const output = '```python\nprint("Hello, World!")\n```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'contains-sql',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-json assertion passes', async () => {
|
|
const output =
|
|
'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} \n\n blah blah';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: containsJsonAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-json assertion passes with multiple json values', async () => {
|
|
const output =
|
|
'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} another {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}}\n\n blah blah';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: containsJsonAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the contains-json assertion fails', async () => {
|
|
const output = 'Not valid JSON';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: containsJsonAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to contain valid JSON',
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-json assertion passes with schema', async () => {
|
|
const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: containsJsonAssertionWithSchema,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-json assertion passes with schema with YAML string', async () => {
|
|
const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: containsJsonAssertionWithSchema,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the contains-json assertion passes with external schema', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains-json',
|
|
value: 'file:///schema.json',
|
|
};
|
|
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
vi.mocked(fs.readFileSync).mockReturnValue(
|
|
JSON.stringify({
|
|
required: ['latitude', 'longitude'],
|
|
type: 'object',
|
|
properties: {
|
|
latitude: {
|
|
type: 'number',
|
|
minimum: -90,
|
|
maximum: 90,
|
|
},
|
|
longitude: {
|
|
type: 'number',
|
|
minimum: -180,
|
|
maximum: 180,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
|
|
const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail contains-json assertion with invalid data against external schema', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains-json',
|
|
value: 'file:///schema.json',
|
|
};
|
|
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
vi.mocked(fs.readFileSync).mockReturnValue(
|
|
JSON.stringify({
|
|
required: ['latitude', 'longitude'],
|
|
type: 'object',
|
|
properties: {
|
|
latitude: {
|
|
type: 'number',
|
|
minimum: -90,
|
|
maximum: 90,
|
|
},
|
|
longitude: {
|
|
type: 'number',
|
|
minimum: -180,
|
|
maximum: 180,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
|
|
const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
|
|
});
|
|
});
|
|
|
|
it('should fail contains-json assertion with predefined schema and invalid data', async () => {
|
|
const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: containsJsonAssertionWithSchema,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toEqual(
|
|
expect.objectContaining({
|
|
pass: false,
|
|
reason:
|
|
'JSON does not conform to the provided schema. Errors: data/latitude must be number',
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should fail when the not-contains-json assertion finds matching schema', async () => {
|
|
const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'not-contains-json',
|
|
value: containsJsonAssertionWithSchema.value,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'Output contains JSON conforming to the provided schema',
|
|
});
|
|
});
|
|
|
|
it('should pass when the not-contains-json assertion finds no matching schema', async () => {
|
|
const output = 'here is the answer\n\n```{"latitude": "not a number", "longitude": -1}```';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'not-contains-json',
|
|
value: containsJsonAssertionWithSchema.value,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the not-contains-json assertion with schema finds no JSON at all', async () => {
|
|
const output = 'here is the answer with no JSON whatsoever';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'not-contains-json',
|
|
value: containsJsonAssertionWithSchema.value,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when the javascript assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptStringAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass a score through when the javascript returns a number', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptStringAssertionWithNumber,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: output.length * 10,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should pass when javascript returns an output string that is smaller than the maximum size threshold', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptBooleanAssertionWithConfig,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: 1.0,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should disregard invalid inputs for assert index', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptBooleanAssertionWithConfig,
|
|
test: {
|
|
assert: [
|
|
{
|
|
type: 'javascript',
|
|
value: 'output.length <= context.config.maximumOutputSize',
|
|
config: {
|
|
maximumOutputSize: 1,
|
|
},
|
|
} as Assertion,
|
|
],
|
|
} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
assertIndex: 45,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: 1.0,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when javascript returns an output string that is larger than the maximum size threshold', async () => {
|
|
const output = 'Expected output with some extra characters';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptBooleanAssertionWithConfig,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
score: 0,
|
|
reason: expect.stringContaining('Custom function returned false'),
|
|
});
|
|
});
|
|
|
|
it('should pass when javascript returns a number above threshold', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptStringAssertionWithNumberAndThreshold,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
score: output.length * 10,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when javascript returns a number below threshold', async () => {
|
|
const output = '';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptStringAssertionWithNumberAndThreshold,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
score: output.length * 10,
|
|
reason: expect.stringContaining('Custom function returned false'),
|
|
});
|
|
});
|
|
|
|
it('should set score when javascript returns false', async () => {
|
|
const output = 'Test output';
|
|
|
|
const assertion: Assertion = {
|
|
type: 'javascript',
|
|
value: 'output.length < 1',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
score: 0,
|
|
reason: expect.stringContaining('Custom function returned false'),
|
|
});
|
|
});
|
|
|
|
it('should fail when the javascript assertion fails', async () => {
|
|
const output = 'Different output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: javascriptStringAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Custom function returned false\noutput === "Expected output"',
|
|
});
|
|
});
|
|
|
|
it('should pass when assertion passes - with vars', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: '{{ foo }}',
|
|
};
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'variable value',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: { vars: { foo: 'Expected output' } } as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
const notContainsAssertion: Assertion = {
|
|
type: 'not-contains',
|
|
value: 'Unexpected output',
|
|
};
|
|
|
|
it('should pass when the not-contains assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notContainsAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the not-contains assertion fails', async () => {
|
|
const output = 'Unexpected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notContainsAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to not contain "Unexpected output"',
|
|
});
|
|
});
|
|
|
|
// Test for icontains assertion
|
|
const containsLowerAssertion: Assertion = {
|
|
type: 'icontains',
|
|
value: 'expected output',
|
|
};
|
|
|
|
it('should pass when the icontains assertion passes', async () => {
|
|
const output = 'EXPECTED OUTPUT';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsLowerAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the icontains assertion fails', async () => {
|
|
const output = 'Different output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsLowerAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to contain "expected output"',
|
|
});
|
|
});
|
|
|
|
// Test for not-icontains assertion
|
|
const notContainsLowerAssertion: Assertion = {
|
|
type: 'not-icontains',
|
|
value: 'unexpected output',
|
|
};
|
|
|
|
it('should pass when the not-icontains assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notContainsLowerAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the not-icontains assertion fails', async () => {
|
|
const output = 'UNEXPECTED OUTPUT';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notContainsLowerAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to not contain "unexpected output"',
|
|
});
|
|
});
|
|
|
|
// Test for contains-any assertion
|
|
const containsAnyAssertion: Assertion = {
|
|
type: 'contains-any',
|
|
value: ['option1', 'option2', 'option3'],
|
|
};
|
|
|
|
it('should pass when the contains-any assertion passes', async () => {
|
|
const output = 'This output contains option1';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsAnyAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the contains-any assertion fails', async () => {
|
|
const output = 'This output does not contain any option';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsAnyAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to contain one of "option1, option2, option3"',
|
|
});
|
|
});
|
|
|
|
it('should pass when the icontains-any assertion passes', async () => {
|
|
const output = 'This output contains OPTION1';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'icontains-any',
|
|
value: ['option1', 'option2', 'option3'],
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the icontains-any assertion fails', async () => {
|
|
const output = 'This output does not contain any option';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'icontains-any',
|
|
value: ['option1', 'option2', 'option3'],
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to contain one of "option1, option2, option3"',
|
|
});
|
|
});
|
|
|
|
// Test for contains-all assertion
|
|
const containsAllAssertion: Assertion = {
|
|
type: 'contains-all',
|
|
value: ['option1', 'option2', 'option3'],
|
|
};
|
|
|
|
it('should pass when the contains-all assertion passes', async () => {
|
|
const output = 'This output contains option1, option2, and option3';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsAllAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the contains-all assertion fails', async () => {
|
|
const output = 'This output contains only option1 and option2';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsAllAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to contain all of [option1, option2, option3]. Missing: [option3]',
|
|
});
|
|
});
|
|
|
|
it('should pass when the icontains-all assertion passes', async () => {
|
|
const output = 'This output contains OPTION1, option2, and opTiOn3';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'icontains-all',
|
|
value: ['option1', 'option2', 'option3'],
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the icontains-all assertion fails', async () => {
|
|
const output = 'This output contains OPTION1, option2, and opTiOn3';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'icontains-all',
|
|
value: ['option1', 'option2', 'option3', 'option4'],
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason:
|
|
'Expected output to contain all of [option1, option2, option3, option4]. Missing: [option4]',
|
|
});
|
|
});
|
|
|
|
// Test for regex assertion
|
|
const containsRegexAssertion: Assertion = {
|
|
type: 'regex',
|
|
value: '\\d{3}-\\d{2}-\\d{4}',
|
|
};
|
|
|
|
it('should pass when the regex assertion passes', async () => {
|
|
const output = 'This output contains 123-45-6789';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsRegexAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the regex assertion fails', async () => {
|
|
const output = 'This output does not contain the pattern';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: containsRegexAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to match regex "\\d{3}-\\d{2}-\\d{4}"',
|
|
});
|
|
});
|
|
|
|
// Test for not-regex assertion
|
|
const notContainsRegexAssertion: Assertion = {
|
|
type: 'not-regex',
|
|
value: '\\d{3}-\\d{2}-\\d{4}',
|
|
};
|
|
|
|
it('should pass when the not-regex assertion passes', async () => {
|
|
const output = 'This output does not contain the pattern';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notContainsRegexAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the not-regex assertion fails', async () => {
|
|
const output = 'This output contains 123-45-6789';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notContainsRegexAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to not match regex "\\d{3}-\\d{2}-\\d{4}"',
|
|
});
|
|
});
|
|
|
|
// Tests for webhook assertion
|
|
const webhookAssertion: Assertion = {
|
|
type: 'webhook',
|
|
value: 'https://example.com/webhook',
|
|
};
|
|
|
|
it('should pass when the webhook assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
vi.mocked(fetchWithRetries).mockImplementation(() =>
|
|
Promise.resolve(
|
|
new Response(JSON.stringify({ pass: true }), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}),
|
|
),
|
|
);
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: webhookAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the webhook assertion fails', async () => {
|
|
const output = 'Different output';
|
|
vi.mocked(fetchWithRetries).mockImplementation(() =>
|
|
Promise.resolve(
|
|
new Response(JSON.stringify({ pass: false }), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}),
|
|
),
|
|
);
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: webhookAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Webhook returned false',
|
|
});
|
|
});
|
|
|
|
it('should fail when the webhook returns an error', async () => {
|
|
const output = 'Expected output';
|
|
|
|
vi.mocked(fetchWithRetries).mockImplementation(() =>
|
|
Promise.resolve(
|
|
new Response('', {
|
|
status: 500,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}),
|
|
),
|
|
);
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: webhookAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Webhook error: Webhook response status: 500',
|
|
});
|
|
});
|
|
|
|
// Test for rouge-n assertion
|
|
const rougeNAssertion: Assertion = {
|
|
type: 'rouge-n',
|
|
value: 'This is the expected output.',
|
|
threshold: 0.75,
|
|
};
|
|
const notRougeNAssertion: Assertion = {
|
|
type: 'not-rouge-n',
|
|
value: 'This is the expected output.',
|
|
threshold: 0.75,
|
|
};
|
|
|
|
it('should pass when the rouge-n assertion passes', async () => {
|
|
const output = 'This is the expected output.';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: rougeNAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'ROUGE-N score 1.00 is greater than or equal to threshold 0.75',
|
|
});
|
|
});
|
|
|
|
it('should fail when the rouge-n assertion fails', async () => {
|
|
const output = 'some different output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: rougeNAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'ROUGE-N score 0.22 is less than threshold 0.75',
|
|
});
|
|
});
|
|
|
|
it('should pass when the not-rouge-n assertion score is below threshold', async () => {
|
|
const output = 'some different output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notRougeNAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'ROUGE-N score 0.22 is less than threshold 0.75',
|
|
});
|
|
expect(result.score).toBeCloseTo(0.78, 2);
|
|
});
|
|
|
|
it('should fail when the not-rouge-n assertion score is above threshold', async () => {
|
|
const output = 'This is the expected output.';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: notRougeNAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'ROUGE-N score 1.00 is greater than or equal to threshold 0.75',
|
|
});
|
|
expect(result.score).toBe(0);
|
|
});
|
|
|
|
// Test for starts-with assertion
|
|
const startsWithAssertion: Assertion = {
|
|
type: 'starts-with',
|
|
value: 'Expected',
|
|
};
|
|
|
|
it('should pass when the starts-with assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: startsWithAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the starts-with assertion fails', async () => {
|
|
const output = 'Different output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: startsWithAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Expected output to start with "Expected"',
|
|
});
|
|
});
|
|
|
|
it('should use the provider from the assertion if it exists', async () => {
|
|
// Assertion grader passes
|
|
const output = 'Expected output';
|
|
const assertion: Assertion = {
|
|
type: 'llm-rubric',
|
|
value: 'Expected output',
|
|
provider: Grader,
|
|
};
|
|
|
|
// Test grader fails
|
|
const BogusGrader = createMockProvider({
|
|
id: 'BogusGrader',
|
|
callApi: vi.fn<ApiProvider['callApi']>().mockRejectedValue(new Error('Should not be called')),
|
|
});
|
|
const test: AtomicTestCase = {
|
|
assert: [assertion],
|
|
options: {
|
|
provider: BogusGrader,
|
|
},
|
|
};
|
|
|
|
// Expect test to pass because assertion grader takes priority
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Test grading output',
|
|
});
|
|
});
|
|
|
|
// Test for levenshtein assertion
|
|
const levenshteinAssertion: Assertion = {
|
|
type: 'levenshtein',
|
|
value: 'Expected output',
|
|
threshold: 5,
|
|
};
|
|
|
|
it('should pass when the levenshtein assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: levenshteinAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the levenshtein assertion fails', async () => {
|
|
const output = 'Different output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: levenshteinAssertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Levenshtein distance 8 is greater than threshold 5',
|
|
});
|
|
});
|
|
|
|
describe('latency assertion', () => {
|
|
it('should pass when the latency assertion passes', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 100,
|
|
},
|
|
latencyMs: 50,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the latency assertion fails', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 100,
|
|
},
|
|
latencyMs: 1000,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Latency 1000ms is greater than threshold 100ms',
|
|
});
|
|
});
|
|
|
|
it('should throw an error when grading result is missing latencyMs', async () => {
|
|
const output = 'Expected output';
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 100,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
}),
|
|
).rejects.toThrow(
|
|
'Latency assertion does not support cached results. Rerun the eval with --no-cache',
|
|
);
|
|
});
|
|
|
|
it('should pass when the latency is 0ms', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 100,
|
|
},
|
|
latencyMs: 0,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should throw an error when threshold is not provided', async () => {
|
|
const output = 'Expected output';
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion: {
|
|
type: 'latency',
|
|
},
|
|
latencyMs: 50,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
}),
|
|
).rejects.toThrow('Latency assertion must have a threshold in milliseconds');
|
|
});
|
|
|
|
it('should handle latency equal to threshold', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 100,
|
|
},
|
|
latencyMs: 100,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should NOT throw error when threshold is 0', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 0,
|
|
},
|
|
latencyMs: 0,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when latency exceeds threshold=0', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'latency',
|
|
threshold: 0,
|
|
},
|
|
latencyMs: 1,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Latency 1ms is greater than threshold 0ms',
|
|
});
|
|
});
|
|
|
|
it('should fail (not-latency) when latency is within threshold', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-latency',
|
|
threshold: 100,
|
|
},
|
|
latencyMs: 50,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Latency 50ms is less than or equal to threshold 100ms',
|
|
});
|
|
});
|
|
|
|
it('should pass (not-latency) when latency exceeds threshold', async () => {
|
|
const output = 'Expected output';
|
|
|
|
const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
|
|
const providerResponse = { output };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-latency',
|
|
threshold: 100,
|
|
},
|
|
latencyMs: 1000,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('perplexity assertion', () => {
|
|
it('should pass when the perplexity assertion passes', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3]; // Dummy logProbs for testing
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity',
|
|
threshold: 2,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the perplexity assertion fails', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3]; // Dummy logProbs for testing
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity',
|
|
threshold: 0.2,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Perplexity 1.28 is greater than threshold 0.2',
|
|
});
|
|
});
|
|
|
|
it('should PASS when no threshold is specified (default behavior)', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
|
|
expect(result.pass).toBe(true); // No threshold = always pass
|
|
});
|
|
|
|
it('should respect threshold=0 as a valid threshold', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity',
|
|
threshold: 0,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
|
|
// Perplexity will be > 0, so with threshold=0, should fail
|
|
expect(result.pass).toBe(false);
|
|
});
|
|
|
|
it('should fail (not-perplexity) when perplexity is within threshold', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-perplexity',
|
|
threshold: 2,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Perplexity 1.28 is less than or equal to threshold 2',
|
|
});
|
|
});
|
|
|
|
it('should pass (not-perplexity) when perplexity exceeds threshold', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-perplexity',
|
|
threshold: 0.2,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('perplexity-score assertion', () => {
|
|
it('should pass when the perplexity-score assertion passes', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity-score',
|
|
threshold: 0.25,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
// Forward variant keeps the raw normalized perplexity as its score.
|
|
expect(result.score).toBeCloseTo(0.4378, 3);
|
|
});
|
|
|
|
it('should fail when the perplexity-score assertion fails', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity-score',
|
|
threshold: 0.5,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Perplexity score 0.44 is less than threshold 0.5',
|
|
});
|
|
});
|
|
|
|
it('should PASS when no threshold is specified for perplexity-score', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity-score',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
|
|
expect(result.pass).toBe(true); // No threshold = always pass
|
|
});
|
|
|
|
it('should respect threshold=0 as valid for perplexity-score', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'perplexity-score',
|
|
threshold: 0,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
|
|
// Perplexity score will be > 0, so should pass with threshold=0
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('should fail (not-perplexity-score) when score is at or above threshold', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-perplexity-score',
|
|
threshold: 0.25,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Perplexity score 0.44 is greater than or equal to threshold 0.25',
|
|
});
|
|
// Score is inverted for the not- variant so it stays correlated with pass.
|
|
expect(result.score).toBeCloseTo(0.5622, 3);
|
|
});
|
|
|
|
it('should pass (not-perplexity-score) when score is below threshold', async () => {
|
|
const logProbs = [-0.2, -0.4, -0.1, -0.3];
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ logProbs }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', logProbs };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-perplexity-score',
|
|
threshold: 0.5,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
// Passing not- assertion contributes a high (inverted) score, not the raw 0.44.
|
|
expect(result.score).toBeCloseTo(0.5622, 3);
|
|
});
|
|
});
|
|
|
|
describe('cost assertion', () => {
|
|
it('should pass when the cost is below the threshold', async () => {
|
|
const cost = 0.0005;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'cost',
|
|
threshold: 0.001,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when the cost exceeds the threshold', async () => {
|
|
const cost = 0.002;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'cost',
|
|
threshold: 0.001,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Cost 0.0020 is greater than threshold 0.001',
|
|
});
|
|
});
|
|
|
|
it('should NOT throw error when threshold is 0', async () => {
|
|
const cost = 0;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'cost',
|
|
threshold: 0,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should fail when cost exceeds threshold=0', async () => {
|
|
const cost = 0.01;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'cost',
|
|
threshold: 0,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Cost 0.010 is greater than threshold 0',
|
|
});
|
|
});
|
|
|
|
it('should fail (not-cost) when the cost is within threshold', async () => {
|
|
const cost = 0.0005;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-cost',
|
|
threshold: 0.001,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Cost 0.00050 is less than or equal to threshold 0.001',
|
|
});
|
|
});
|
|
|
|
it('should pass (not-cost) when the cost exceeds threshold', async () => {
|
|
const cost = 0.002;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'not-cost',
|
|
threshold: 0.001,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should throw when no threshold is provided', async () => {
|
|
const cost = 0.0005;
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output', cost };
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'cost',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
}),
|
|
).rejects.toThrow('Cost assertion must have a threshold');
|
|
});
|
|
|
|
it('should throw when the provider does not return cost', async () => {
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ output: 'Some output' }),
|
|
} as unknown as ApiProvider;
|
|
const providerResponse = { output: 'Some output' };
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider,
|
|
assertion: {
|
|
type: 'cost',
|
|
threshold: 0.001,
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse,
|
|
}),
|
|
).rejects.toThrow('Cost assertion does not support providers that do not return cost');
|
|
});
|
|
});
|
|
|
|
describe('Similarity assertion', () => {
|
|
beforeEach(() => {
|
|
vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockImplementation((text) => {
|
|
if (text === 'Test output' || text.startsWith('Similar output')) {
|
|
return Promise.resolve({
|
|
embedding: [1, 0, 0],
|
|
tokenUsage: { total: 5, prompt: 2, completion: 3 },
|
|
});
|
|
} else if (text.startsWith('Different output')) {
|
|
return Promise.resolve({
|
|
embedding: [0, 1, 0],
|
|
tokenUsage: { total: 5, prompt: 2, completion: 3 },
|
|
});
|
|
}
|
|
return Promise.reject(new Error('Unexpected input'));
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
it('should pass for a similar assertion with a string value', async () => {
|
|
const output = 'Test output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'similar',
|
|
value: 'Similar output',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Similarity 1.00 is greater than or equal to threshold 0.75',
|
|
});
|
|
});
|
|
|
|
it('should fail for a similar assertion with a string value', async () => {
|
|
const output = 'Test output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'similar',
|
|
value: 'Different output',
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'Similarity 0.00 is less than threshold 0.75',
|
|
});
|
|
});
|
|
|
|
it('should pass for a similar assertion with an array of string values', async () => {
|
|
const output = 'Test output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'similar',
|
|
value: ['Similar output 1', 'Different output 1'],
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Similarity 1.00 is greater than or equal to threshold 0.75',
|
|
});
|
|
});
|
|
|
|
it('should fail for a similar assertion with an array of string values', async () => {
|
|
const output = 'Test output';
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion: {
|
|
type: 'similar',
|
|
value: ['Different output 1', 'Different output 2'],
|
|
},
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
reason: 'None of the provided values met the similarity threshold',
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('is-xml', () => {
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost: 0.001 }),
|
|
} as unknown as ApiProvider;
|
|
|
|
it('should pass when the output is valid XML', async () => {
|
|
const output = '<root><child>Content</child></root>';
|
|
const assertion: Assertion = { type: 'is-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should fail when the output is not valid XML', async () => {
|
|
const output = '<root><child>Content</child></root';
|
|
const assertion: Assertion = { type: 'is-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: false,
|
|
score: 0,
|
|
reason: expect.stringMatching(/XML parsing failed/),
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should pass when required elements are present', async () => {
|
|
const output =
|
|
'<analysis><classification>T-shirt</classification><color>Red</color></analysis>';
|
|
const assertion: Assertion = {
|
|
type: 'is-xml',
|
|
value: 'analysis.classification,analysis.color',
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should fail when required elements are missing', async () => {
|
|
const output = '<analysis><classification>T-shirt</classification></analysis>';
|
|
const assertion: Assertion = {
|
|
type: 'is-xml',
|
|
value: 'analysis.classification,analysis.color',
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'XML is missing required elements: analysis.color',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should pass when nested elements are present', async () => {
|
|
const output =
|
|
'<root><parent><child><grandchild>Content</grandchild></child></parent></root>';
|
|
const assertion: Assertion = {
|
|
type: 'is-xml',
|
|
value: 'root.parent.child.grandchild',
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should handle inverse assertion correctly', async () => {
|
|
const output = 'This is not XML';
|
|
const assertion: Assertion = { type: 'not-is-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate non-XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should pass when required elements are specified as an array', async () => {
|
|
const output = '<root><element1>Content1</element1><element2>Content2</element2></root>';
|
|
const assertion: Assertion = {
|
|
type: 'is-xml',
|
|
value: ['root.element1', 'root.element2'],
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should pass when required elements are specified as an object', async () => {
|
|
const output = '<root><element1>Content1</element1><element2>Content2</element2></root>';
|
|
const assertion: Assertion = {
|
|
type: 'contains-xml',
|
|
value: {
|
|
requiredElements: ['root.element1', 'root.element2'],
|
|
},
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should throw an error when xml assertion value is invalid', async () => {
|
|
const output = '<root><element1>Content1</element1><element2>Content2</element2></root>';
|
|
const assertion: Assertion = {
|
|
type: 'is-xml',
|
|
value: { invalidKey: ['root.element1', 'root.element2'] },
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Generate XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
}),
|
|
).rejects.toThrow('xml assertion must contain a string, array value, or no value');
|
|
});
|
|
|
|
it('should handle multiple XML blocks in contains-xml assertion', async () => {
|
|
const output = 'Some text <xml1>content1</xml1> more text <xml2>content2</xml2>';
|
|
const assertion: Assertion = {
|
|
type: 'contains-xml',
|
|
value: ['xml1', 'xml2'],
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text with multiple XML blocks',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('contains-xml', () => {
|
|
const provider = {
|
|
callApi: vi.fn().mockResolvedValue({ cost: 0.001 }),
|
|
} as unknown as ApiProvider;
|
|
it('should pass when the output contains valid XML', async () => {
|
|
const output = 'Some text before <root><child>Content</child></root> and after';
|
|
const assertion: Assertion = { type: 'contains-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text with XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should fail when the output does not contain valid XML', async () => {
|
|
const output = 'This is just plain text without any XML';
|
|
const assertion: Assertion = { type: 'contains-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text without XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'No XML content found in the output',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should pass when required elements are present in the XML', async () => {
|
|
const output =
|
|
'Before <analysis><classification>T-shirt</classification><color>Red</color></analysis> After';
|
|
const assertion: Assertion = {
|
|
type: 'contains-xml',
|
|
value: 'analysis.classification,analysis.color',
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text with specific XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should fail when required elements are missing in the XML', async () => {
|
|
const output = 'Before <analysis><classification>T-shirt</classification></analysis> After';
|
|
const assertion: Assertion = {
|
|
type: 'contains-xml',
|
|
value: 'analysis.classification,analysis.color',
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text with specific XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'No valid XML content found matching the requirements',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should pass when nested elements are present in the XML', async () => {
|
|
const output =
|
|
'Start <root><parent><child><grandchild>Content</grandchild></child></parent></root> End';
|
|
const assertion: Assertion = {
|
|
type: 'contains-xml',
|
|
value: 'root.parent.child.grandchild',
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text with nested XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should handle inverse assertion correctly', async () => {
|
|
const output = 'This is just plain text without any XML';
|
|
const assertion: Assertion = { type: 'not-contains-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text without XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Assertion passed',
|
|
assertion,
|
|
});
|
|
});
|
|
|
|
it('should fail inverse assertion when XML is present', async () => {
|
|
const output = 'Some text with <xml>content</xml> in it';
|
|
const assertion: Assertion = { type: 'not-contains-xml' };
|
|
|
|
const result = await runAssertion({
|
|
prompt: 'Generate text without XML',
|
|
provider,
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: false,
|
|
score: 0,
|
|
reason: 'XML is valid and contains all required elements',
|
|
assertion,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('context-relevance assertion', () => {
|
|
it('should pass when all required vars are present', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-relevance',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Mocked reason',
|
|
assertion,
|
|
metadata: {
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
});
|
|
});
|
|
|
|
it('should use resolved vars for context-based grading', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-relevance',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
context: 'file://docs/france.md',
|
|
},
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
assertion,
|
|
test,
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
providerResponse: { output: 'Some output' },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
metadata: {
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
});
|
|
});
|
|
|
|
it('should throw an error when vars object is missing', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-relevance',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
}),
|
|
).rejects.toThrow('context-relevance assertion requires a test with variables');
|
|
});
|
|
|
|
it('should throw an error when query var is missing', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-relevance',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
}),
|
|
).rejects.toThrow(
|
|
'context-relevance assertion requires a "query" variable with the user question',
|
|
);
|
|
});
|
|
|
|
it('should throw an error when context var is missing', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-relevance',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
}),
|
|
).rejects.toThrow(
|
|
'Context is required for context-based assertions. Provide either a "context" variable (string or array of strings) in your test case or use "contextTransform" to extract context from the provider response.',
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('context-faithfulness assertion', () => {
|
|
it('should pass when all required vars are present', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-faithfulness',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
};
|
|
|
|
const result = await runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'The capital of France is Paris.' },
|
|
});
|
|
|
|
expect(result).toEqual({
|
|
pass: true,
|
|
score: 1,
|
|
reason: 'Mocked reason',
|
|
assertion,
|
|
metadata: {
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
});
|
|
});
|
|
|
|
it('should throw an error when vars object is missing', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-faithfulness',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
}),
|
|
).rejects.toThrow('context-faithfulness assertion requires a test with variables');
|
|
});
|
|
|
|
it('should throw an error when query var is missing', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-faithfulness',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
}),
|
|
).rejects.toThrow(
|
|
'context-faithfulness assertion requires a "query" variable with the user question',
|
|
);
|
|
});
|
|
|
|
it('should throw an error when context var is missing', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-faithfulness',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Some output' },
|
|
}),
|
|
).rejects.toThrow(
|
|
'Context is required for context-based assertions. Provide either a "context" variable (string or array of strings) in your test case or use "contextTransform" to extract context from the provider response.',
|
|
);
|
|
});
|
|
|
|
it('should throw an error when output is not a string', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'context-faithfulness',
|
|
threshold: 0.7,
|
|
};
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
query: 'What is the capital of France?',
|
|
context: 'Paris is the capital of France.',
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: { some: 'object' } },
|
|
}),
|
|
).rejects.toThrow('context-faithfulness assertion requires string output from the provider');
|
|
});
|
|
});
|
|
|
|
describe('assertion transform with metadata', () => {
|
|
it('should pass metadata to assertion transform when available', async () => {
|
|
const output = 'Test output';
|
|
const metadata = { key: 'value', nested: { data: 123 } };
|
|
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'Metadata: {"key":"value","nested":{"data":123}}',
|
|
transform: 'context.metadata ? `Metadata: ${JSON.stringify(context.metadata)}` : output',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output, metadata },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should handle transform when metadata is undefined', async () => {
|
|
const output = 'Test output';
|
|
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'No metadata',
|
|
transform: 'context.metadata ? "Has metadata" : "No metadata"',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should handle transform when providerResponse has no metadata', async () => {
|
|
const output = 'Test output';
|
|
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'No metadata',
|
|
transform: 'context.metadata ? "Has metadata" : "No metadata"',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output }, // No metadata property
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should handle empty metadata object', async () => {
|
|
const output = 'Test output';
|
|
const metadata = {};
|
|
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'Empty metadata',
|
|
transform:
|
|
'(context.metadata && Object.keys(context.metadata).length === 0) ? "Empty metadata" : output',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output, metadata },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
|
|
it('should preserve existing context properties when adding metadata', async () => {
|
|
const output = 'Test output';
|
|
const metadata = { responseTime: 100 };
|
|
const testVars = { userInput: 'test input' };
|
|
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'All context properties present',
|
|
transform:
|
|
'(context.vars && context.prompt && context.metadata) ? "All context properties present" : "Missing context properties"',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'test prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: { vars: testVars } as AtomicTestCase,
|
|
providerResponse: { output, metadata },
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
pass: true,
|
|
reason: 'Assertion passed',
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('inline function transforms (Node.js package)', () => {
|
|
it('should support a function as assertion transform', async () => {
|
|
const output = 'hello world';
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'HELLO WORLD',
|
|
transform: (output) => String(output).toUpperCase(),
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result.pass).toBe(true);
|
|
expect(result.reason).toBe('Assertion passed');
|
|
});
|
|
|
|
it('should pass context to inline function transform', async () => {
|
|
const output = 'raw output';
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'search, calculate',
|
|
transform: (_output, context: any) => {
|
|
const tools = context.metadata?.toolCalls ?? [];
|
|
return tools.map((t: any) => t.name).join(', ');
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: {
|
|
output,
|
|
metadata: { toolCalls: [{ name: 'search' }, { name: 'calculate' }] },
|
|
},
|
|
});
|
|
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('should support async function as assertion transform', async () => {
|
|
const output = 'hello';
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'hello async',
|
|
transform: async (output) => output + ' async',
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output },
|
|
});
|
|
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('should surface synchronous errors thrown by an inline function transform', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'anything',
|
|
transform: () => {
|
|
throw new Error('inline transform boom');
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output: 'hello world' },
|
|
}),
|
|
).rejects.toThrow('inline transform boom');
|
|
});
|
|
|
|
it('should surface async rejections from an inline function transform', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'anything',
|
|
transform: async () => {
|
|
throw new Error('async transform boom');
|
|
},
|
|
};
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output: 'hello world' },
|
|
}),
|
|
).rejects.toThrow('async transform boom');
|
|
});
|
|
});
|
|
|
|
describe('file references', () => {
|
|
it('should handle file reference in string value', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'equals',
|
|
value: 'file://expected_output.txt',
|
|
};
|
|
|
|
const expectedContent = 'Expected output';
|
|
vi.mocked(fs.readFileSync).mockReturnValue(expectedContent);
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/expected_output.txt');
|
|
vi.mocked(path.extname).mockReturnValue('.txt');
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output: 'Expected output' },
|
|
});
|
|
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/expected_output.txt', 'utf8');
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('should handle file references in array values', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains-any',
|
|
value: ['The expected output', 'string output', 'file://my_expected_output.txt'],
|
|
};
|
|
|
|
const fileContent = 'file content';
|
|
vi.mocked(fs.readFileSync).mockReturnValue(fileContent);
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/my_expected_output.txt');
|
|
vi.mocked(path.extname).mockReturnValue('.txt');
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output: 'file content' },
|
|
}),
|
|
).resolves.toEqual(
|
|
expect.objectContaining({
|
|
pass: true,
|
|
}),
|
|
);
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/my_expected_output.txt', 'utf8');
|
|
|
|
await expect(
|
|
runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output: 'string output' },
|
|
}),
|
|
).resolves.toEqual(
|
|
expect.objectContaining({
|
|
pass: true,
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should handle file reference in object value', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'is-json',
|
|
value: 'file://schema.json',
|
|
};
|
|
|
|
const schemaContent = JSON.stringify({
|
|
type: 'object',
|
|
properties: {
|
|
key: { type: 'string' },
|
|
},
|
|
});
|
|
vi.mocked(fs.readFileSync).mockReturnValue(schemaContent);
|
|
vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
|
|
vi.mocked(path.extname).mockReturnValue('.json');
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
assertion,
|
|
test: {} as AtomicTestCase,
|
|
providerResponse: { output: '{"key": "value"}' },
|
|
});
|
|
|
|
expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('Rendered assertion values', () => {
|
|
it('should store rendered assertion value in metadata when variables are substituted', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains',
|
|
value: 'User said: {{ user_input }}',
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
user_input: 'hello world',
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'User said: hello world' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
expect(result.metadata?.renderedAssertionValue).toBe('User said: hello world');
|
|
expect(result.assertion?.value).toBe('User said: {{ user_input }}');
|
|
});
|
|
|
|
it('should store rendered assertion value with loops', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains',
|
|
value:
|
|
'{% for item in items %}{{ item.name }}{% if not loop.last %}, {% endif %}{% endfor %}',
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
items: [{ name: 'apple' }, { name: 'banana' }, { name: 'cherry' }],
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'apple, banana, cherry' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
expect(result.metadata?.renderedAssertionValue).toBe('apple, banana, cherry');
|
|
});
|
|
|
|
it('should not store rendered value when template equals original', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains',
|
|
value: 'Static text without variables',
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
user_input: 'hello',
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Static text without variables' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
expect(result.metadata?.renderedAssertionValue).toBeUndefined();
|
|
});
|
|
|
|
it('should store rendered value for complex template matching GitHub issue example', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains',
|
|
value: `Did {{ agent_name }} select the expected tools across the conversation?
|
|
|
|
{% for turn in turns %}
|
|
{% if turn.expected_tool %}Turn {{ loop.index }}: Expected tool "{{ turn.expected_tool }}"
|
|
{% endif %}
|
|
{% endfor %}`,
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
agent_name: 'Virtual Assistant',
|
|
turns: [
|
|
{ expected_tool: 'preview_create' },
|
|
{ expected_tool: 'preview_create' },
|
|
{ expected_tool: 'execute_create' },
|
|
],
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'some output' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
expect(result.metadata?.renderedAssertionValue).toContain('Virtual Assistant');
|
|
expect(result.metadata?.renderedAssertionValue).toContain(
|
|
'Turn 1: Expected tool "preview_create"',
|
|
);
|
|
expect(result.metadata?.renderedAssertionValue).toContain(
|
|
'Turn 2: Expected tool "preview_create"',
|
|
);
|
|
expect(result.metadata?.renderedAssertionValue).toContain(
|
|
'Turn 3: Expected tool "execute_create"',
|
|
);
|
|
});
|
|
|
|
it('should store rendered value for javascript assertions', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'javascript',
|
|
value: "output.includes('{{ expected_value }}')",
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
expected_value: 'hello',
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'hello world' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
expect(result.metadata?.renderedAssertionValue).toBe("output.includes('hello')");
|
|
});
|
|
|
|
it('should store rendered value even when assertion fails', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains',
|
|
value: 'Expected: {{ value }}',
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
value: 'foo',
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'Different output - no match' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
// Assertion should fail (output doesn't contain "Expected: foo")
|
|
expect(result.pass).toBe(false);
|
|
// But rendered value should still be captured in metadata
|
|
expect(result.metadata?.renderedAssertionValue).toBe('Expected: foo');
|
|
// Original template should remain in assertion
|
|
expect(result.assertion?.value).toBe('Expected: {{ value }}');
|
|
});
|
|
|
|
it('should preserve existing metadata when adding renderedAssertionValue', async () => {
|
|
const assertion: Assertion = {
|
|
type: 'contains',
|
|
value: 'User: {{ name }}',
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: {
|
|
name: 'Alice',
|
|
},
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'User: Alice' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
// Should have rendered value
|
|
expect(result.metadata?.renderedAssertionValue).toBe('User: Alice');
|
|
// Original template in assertion
|
|
expect(result.assertion?.value).toBe('User: {{ name }}');
|
|
// Metadata object should exist and be extensible
|
|
expect(result.metadata).toBeTruthy();
|
|
});
|
|
|
|
// Regression for https://github.com/promptfoo/promptfoo/issues/7861: the
|
|
// reporter claimed defaultTest llm-rubric rubrics sent raw {{var}} text to
|
|
// the grader. In reality the assertion pipeline renders the value before
|
|
// grading — so the grading LLM DOES receive the substituted string. Prove
|
|
// this by letting the assertion run all the way through a real grader
|
|
// provider and inspecting the prompt it received.
|
|
it('renders llm-rubric value with vars before calling the grader', async () => {
|
|
const capturedPrompt = vi.fn<ApiProvider['callApi']>(async () => ({
|
|
output: JSON.stringify({ pass: true, score: 1, reason: 'graded' }),
|
|
}));
|
|
const capturingGrader = createMockProvider({
|
|
id: 'capturing-grader',
|
|
callApi: capturedPrompt,
|
|
});
|
|
|
|
const assertion: Assertion = {
|
|
type: 'llm-rubric',
|
|
value: 'Does the output correctly reference the input: {{myVar}}?',
|
|
provider: capturingGrader,
|
|
};
|
|
|
|
const test: AtomicTestCase = {
|
|
vars: { myVar: 'hello world' },
|
|
};
|
|
|
|
const result: GradingResult = await runAssertion({
|
|
prompt: 'Some prompt',
|
|
assertion,
|
|
test,
|
|
providerResponse: { output: 'static model output' },
|
|
provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
|
|
});
|
|
|
|
const gradingPrompt = capturedPrompt.mock.calls[0]?.[0] as string;
|
|
expect(gradingPrompt).toContain('hello world');
|
|
expect(gradingPrompt).not.toContain('{{myVar}}');
|
|
|
|
expect(result.metadata?.renderedAssertionValue).toBe(
|
|
'Does the output correctly reference the input: hello world?',
|
|
);
|
|
expect(result.assertion?.value).toBe(
|
|
'Does the output correctly reference the input: {{myVar}}?',
|
|
);
|
|
});
|
|
});
|
|
});
|