chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { PromptConfigSchema, PromptSchema } from '../../src/validators/prompts';
|
||||
|
||||
describe('PromptConfigSchema', () => {
|
||||
it('should accept empty object', () => {
|
||||
const result = PromptConfigSchema.safeParse({});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual({});
|
||||
});
|
||||
|
||||
it('should accept valid prefix and suffix', () => {
|
||||
const input = { prefix: 'You are a helpful assistant.', suffix: 'Please respond.' };
|
||||
const result = PromptConfigSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual(input);
|
||||
});
|
||||
|
||||
it('should accept only prefix', () => {
|
||||
const input = { prefix: 'System: ' };
|
||||
const result = PromptConfigSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual(input);
|
||||
});
|
||||
|
||||
it('should accept only suffix', () => {
|
||||
const input = { suffix: '\nEnd.' };
|
||||
const result = PromptConfigSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual(input);
|
||||
});
|
||||
|
||||
it('should accept empty string values', () => {
|
||||
const input = { prefix: '', suffix: '' };
|
||||
const result = PromptConfigSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual(input);
|
||||
});
|
||||
|
||||
it('should reject non-string prefix', () => {
|
||||
const result = PromptConfigSchema.safeParse({ prefix: 123 });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject non-string suffix', () => {
|
||||
const result = PromptConfigSchema.safeParse({ suffix: true });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PromptSchema', () => {
|
||||
it('should accept valid prompt with required fields', () => {
|
||||
const input = { raw: 'Hello {{name}}', label: 'Greeting prompt' };
|
||||
const result = PromptSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toMatchObject(input);
|
||||
});
|
||||
|
||||
it('should accept prompt with all fields', () => {
|
||||
const mockFn = async () => 'response';
|
||||
const input = {
|
||||
id: 'prompt-1',
|
||||
raw: 'Tell me about {{topic}}',
|
||||
display: 'Topic prompt (deprecated)',
|
||||
label: 'Topic prompt',
|
||||
function: mockFn,
|
||||
config: { temperature: 0.7 },
|
||||
};
|
||||
const result = PromptSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveProperty('id', 'prompt-1');
|
||||
expect(result.data).toHaveProperty('raw', 'Tell me about {{topic}}');
|
||||
expect(result.data).toHaveProperty('label', 'Topic prompt');
|
||||
expect(result.data).toHaveProperty('function');
|
||||
expect(result.data).toHaveProperty('config');
|
||||
});
|
||||
|
||||
it('should reject missing raw field', () => {
|
||||
const result = PromptSchema.safeParse({ label: 'No raw' });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject missing label field', () => {
|
||||
const result = PromptSchema.safeParse({ raw: 'Hello' });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject empty object', () => {
|
||||
const result = PromptSchema.safeParse({});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should accept prompt with optional id', () => {
|
||||
const result = PromptSchema.safeParse({
|
||||
id: 'test-id',
|
||||
raw: 'prompt text',
|
||||
label: 'test label',
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveProperty('id', 'test-id');
|
||||
});
|
||||
|
||||
it('should accept prompt without optional fields', () => {
|
||||
const result = PromptSchema.safeParse({
|
||||
raw: 'minimal prompt',
|
||||
label: 'minimal',
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).not.toHaveProperty('id');
|
||||
expect(result.data).not.toHaveProperty('display');
|
||||
expect(result.data).not.toHaveProperty('function');
|
||||
});
|
||||
|
||||
it('should reject non-function for function field', () => {
|
||||
const result = PromptSchema.safeParse({
|
||||
raw: 'test',
|
||||
label: 'test',
|
||||
function: 'not-a-function',
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should accept any config value', () => {
|
||||
const result = PromptSchema.safeParse({
|
||||
raw: 'test',
|
||||
label: 'test',
|
||||
config: { nested: { deep: true }, array: [1, 2, 3] },
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data?.config).toEqual({ nested: { deep: true }, array: [1, 2, 3] });
|
||||
});
|
||||
|
||||
it('should reject non-string raw value', () => {
|
||||
const result = PromptSchema.safeParse({ raw: 42, label: 'test' });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject non-string label value', () => {
|
||||
const result = PromptSchema.safeParse({ raw: 'test', label: 123 });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { ProviderOptionsSchema, ProviderSchema } from '../../src/validators/providers';
|
||||
import { createMockProvider } from '../factories/provider';
|
||||
|
||||
describe('ProviderOptionsSchema', () => {
|
||||
it('should filter unknown keys without erroring', () => {
|
||||
const input = {
|
||||
id: 'test-provider',
|
||||
label: 'Test Provider',
|
||||
unknownField: 'this should be filtered',
|
||||
anotherUnknown: 123,
|
||||
};
|
||||
|
||||
const result = ProviderOptionsSchema.safeParse(input);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveProperty('id', 'test-provider');
|
||||
expect(result.data).toHaveProperty('label', 'Test Provider');
|
||||
expect(result.data).not.toHaveProperty('unknownField');
|
||||
expect(result.data).not.toHaveProperty('anotherUnknown');
|
||||
});
|
||||
|
||||
it('should accept valid provider options', () => {
|
||||
const input = {
|
||||
id: 'test-provider',
|
||||
label: 'Test Provider',
|
||||
config: { temperature: 0.7 },
|
||||
prompts: ['prompt1', 'prompt2'],
|
||||
transform: 'output.toLowerCase()',
|
||||
delay: 1000,
|
||||
};
|
||||
|
||||
const result = ProviderOptionsSchema.safeParse(input);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual(input);
|
||||
});
|
||||
|
||||
it('should accept empty object', () => {
|
||||
const result = ProviderOptionsSchema.safeParse({});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('ProviderSchema union', () => {
|
||||
it('should match ApiProviderSchema before ProviderOptionsSchema when callApi is present', () => {
|
||||
const input = createMockProvider({
|
||||
id: 'custom-provider',
|
||||
label: 'Custom Provider',
|
||||
});
|
||||
|
||||
const result = ProviderSchema.safeParse(input);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
// callApi should be preserved because ApiProviderSchema matches first
|
||||
expect(result.data).toHaveProperty('callApi');
|
||||
expect(result.data).toHaveProperty('id');
|
||||
expect(result.data).toHaveProperty('label', 'Custom Provider');
|
||||
});
|
||||
|
||||
it('should match ProviderOptionsSchema when no callApi function', () => {
|
||||
const input = {
|
||||
id: 'test-provider',
|
||||
label: 'Test Provider',
|
||||
unknownField: 'should be filtered',
|
||||
};
|
||||
|
||||
const result = ProviderSchema.safeParse(input);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveProperty('id', 'test-provider');
|
||||
expect(result.data).toHaveProperty('label', 'Test Provider');
|
||||
// unknownField should be filtered by ProviderOptionsSchema
|
||||
expect(result.data).not.toHaveProperty('unknownField');
|
||||
});
|
||||
|
||||
it('should accept string provider', () => {
|
||||
const result = ProviderSchema.safeParse('openai:gpt-4');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toBe('openai:gpt-4');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,976 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
ALIASED_PLUGIN_MAPPINGS,
|
||||
ALIASED_PLUGINS,
|
||||
COLLECTIONS,
|
||||
DEFAULT_NUM_TESTS_PER_PLUGIN,
|
||||
DEFAULT_PLUGINS,
|
||||
FOUNDATION_PLUGINS,
|
||||
HARM_PLUGINS,
|
||||
PII_PLUGINS,
|
||||
ALL_PLUGINS as REDTEAM_ALL_PLUGINS,
|
||||
Severity,
|
||||
TEEN_SAFETY_PLUGINS,
|
||||
} from '../../src/redteam/constants';
|
||||
import {
|
||||
pluginOptions,
|
||||
RedteamConfigSchema,
|
||||
RedteamStrategySchema,
|
||||
strategyIdSchema,
|
||||
} from '../../src/validators/redteam';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, 'debug').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('RedteamConfigSchema', () => {
|
||||
it('should validate basic config', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: ['default'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins).toBeDefined();
|
||||
expect(result.strategies).toBeDefined();
|
||||
});
|
||||
|
||||
it('should expand default plugins', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.length).toBeGreaterThan(0);
|
||||
expect(result.plugins?.every((p: any) => Array.from(DEFAULT_PLUGINS).includes(p.id))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('should expand foundation plugins', () => {
|
||||
const config = {
|
||||
plugins: ['foundation'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.length).toBeGreaterThan(0);
|
||||
expect(result.plugins?.every((p: any) => Array.from(FOUNDATION_PLUGINS).includes(p.id))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('should expand harmful plugins', () => {
|
||||
const config = {
|
||||
plugins: ['harmful'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.length).toBeGreaterThan(0);
|
||||
expect(result.plugins?.every((p: any) => Object.keys(HARM_PLUGINS).includes(p.id))).toBe(true);
|
||||
});
|
||||
|
||||
it('should expand PII plugins', () => {
|
||||
const config = {
|
||||
plugins: ['pii'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.length).toBeGreaterThan(0);
|
||||
expect(result.plugins?.every((p: any) => Array.from(PII_PLUGINS).includes(p.id))).toBe(true);
|
||||
});
|
||||
|
||||
it('should expand teen safety plugins', () => {
|
||||
const config = {
|
||||
plugins: ['teen-safety'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.map((plugin) => plugin.id).sort()).toEqual(
|
||||
[...TEEN_SAFETY_PLUGINS].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle plugin with config and severity', () => {
|
||||
const config = {
|
||||
plugins: [
|
||||
{
|
||||
id: 'file://test-plugin.js',
|
||||
config: { key: 'value' },
|
||||
severity: Severity.High,
|
||||
numTests: DEFAULT_NUM_TESTS_PER_PLUGIN,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.[0]).toEqual({
|
||||
id: 'file://test-plugin.js',
|
||||
config: { key: 'value' },
|
||||
severity: Severity.High,
|
||||
numTests: DEFAULT_NUM_TESTS_PER_PLUGIN,
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle aliased plugins', () => {
|
||||
const config = {
|
||||
plugins: [Object.keys(ALIASED_PLUGIN_MAPPINGS)[0]],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should validate custom file:// plugins', () => {
|
||||
const config = {
|
||||
plugins: ['file:///path/to/plugin.js'],
|
||||
};
|
||||
|
||||
expect(() => RedteamConfigSchema.parse(config)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should reject invalid plugin names', () => {
|
||||
const config = {
|
||||
plugins: ['invalid-plugin-name'],
|
||||
};
|
||||
|
||||
expect(() => RedteamConfigSchema.parse(config)).toThrow(/Invalid plugin id/);
|
||||
});
|
||||
|
||||
it('should handle numTests override', () => {
|
||||
const config = {
|
||||
numTests: 5,
|
||||
plugins: ['file://test-plugin.js'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.[0].numTests).toBe(5);
|
||||
});
|
||||
|
||||
it('should inherit top-level numTests for object plugins that omit numTests', () => {
|
||||
const config = {
|
||||
numTests: 17,
|
||||
plugins: [{ id: 'file://test-plugin.js' }],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.[0].numTests).toBe(17);
|
||||
});
|
||||
|
||||
it('should use default numTests when not specified', () => {
|
||||
const config = {
|
||||
plugins: [
|
||||
{
|
||||
id: 'file://test-plugin.js',
|
||||
numTests: DEFAULT_NUM_TESTS_PER_PLUGIN,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.[0].numTests).toBe(DEFAULT_NUM_TESTS_PER_PLUGIN);
|
||||
});
|
||||
|
||||
it('should deduplicate plugins', () => {
|
||||
const config = {
|
||||
plugins: ['file://test-plugin.js', 'file://test-plugin.js'],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should handle collection plugins', () => {
|
||||
const config = {
|
||||
plugins: COLLECTIONS,
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should validate RedteamPluginObjectSchema', () => {
|
||||
const validPlugin = {
|
||||
id: 'file://test-plugin.js',
|
||||
numTests: 5,
|
||||
config: { key: 'value' },
|
||||
severity: Severity.High,
|
||||
};
|
||||
|
||||
const config = {
|
||||
plugins: [validPlugin],
|
||||
};
|
||||
|
||||
expect(() => RedteamConfigSchema.parse(config)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle plugin with severity but no config', () => {
|
||||
const config = {
|
||||
plugins: [
|
||||
{
|
||||
id: 'file://test-plugin.js',
|
||||
severity: Severity.Low,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.[0].severity).toBe(Severity.Low);
|
||||
expect(result.plugins?.[0].config).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should preserve severity when expanding collections', () => {
|
||||
const config = {
|
||||
plugins: [
|
||||
{
|
||||
id: 'foundation',
|
||||
severity: Severity.Critical,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins?.every((p) => p.severity === Severity.Critical)).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle strategy with config', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [{ id: 'default', config: { key: 'value' } }],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.strategies?.[0]).toEqual(expect.objectContaining({ config: { key: 'value' } }));
|
||||
});
|
||||
|
||||
it('should handle custom file:// strategies', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: ['file:///path/to/strategy.js'],
|
||||
};
|
||||
|
||||
expect(() => RedteamConfigSchema.parse(config)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle empty plugins array', () => {
|
||||
const config = {
|
||||
plugins: [],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.plugins).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle config with all optional fields', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: ['basic'],
|
||||
numTests: 10,
|
||||
language: 'en',
|
||||
injectVar: 'prompt',
|
||||
purpose: 'Testing LLM robustness',
|
||||
entities: ['Company A', 'Product B'],
|
||||
maxConcurrency: 5,
|
||||
delay: 1000,
|
||||
provider: 'openai:gpt-4',
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
expect(result.numTests).toBe(10);
|
||||
expect(result.language).toBe('en');
|
||||
expect(result.injectVar).toBe('prompt');
|
||||
expect(result.purpose).toBe('Testing LLM robustness');
|
||||
expect(result.entities).toEqual(['Company A', 'Product B']);
|
||||
expect(result.delay).toBe(1000);
|
||||
expect(result.provider).toBe('openai:gpt-4');
|
||||
});
|
||||
|
||||
it('should reject invalid numTests values', () => {
|
||||
const invalidConfigs = [
|
||||
{ plugins: ['default'], numTests: 0 }, // Zero
|
||||
{ plugins: ['default'], numTests: -5 }, // Negative
|
||||
{ plugins: ['default'], numTests: 1.5 }, // Decimal
|
||||
{ plugins: ['default'], numTests: 'five' as any }, // String
|
||||
];
|
||||
|
||||
invalidConfigs.forEach((config) => {
|
||||
expect(() => RedteamConfigSchema.parse(config)).toThrow(z.ZodError);
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject invalid delay values', () => {
|
||||
const invalidConfigs = [
|
||||
{ plugins: ['default'], delay: -100 }, // Negative delay
|
||||
{ plugins: ['default'], delay: 'slow' as any }, // String
|
||||
];
|
||||
|
||||
invalidConfigs.forEach((config) => {
|
||||
expect(() => RedteamConfigSchema.parse(config)).toThrow(z.ZodError);
|
||||
});
|
||||
});
|
||||
|
||||
it('should properly transform and deduplicate complex plugin configurations', () => {
|
||||
const config = {
|
||||
plugins: [
|
||||
'default',
|
||||
'default', // Duplicate
|
||||
{ id: 'contracts', numTests: 10 },
|
||||
{ id: 'contracts', numTests: 10 }, // Exact duplicate
|
||||
{ id: 'contracts', numTests: 5 }, // Different numTests - this one will be kept
|
||||
'file://custom1.js',
|
||||
'file://custom1.js', // Duplicate file
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// The transform function deduplicates based on key: `${id}:${JSON.stringify(config)}:${severity || ''}`
|
||||
// So contracts with different numTests are different plugins
|
||||
const contractsPlugins = result.plugins?.filter((p) => p.id === 'contracts') || [];
|
||||
expect(contractsPlugins.length).toBeGreaterThan(0);
|
||||
|
||||
// File plugins are deduplicated
|
||||
const customPlugins = result.plugins?.filter((p) => p.id === 'file://custom1.js') || [];
|
||||
expect(customPlugins).toHaveLength(1); // Deduplicated
|
||||
|
||||
// Default plugins should be expanded
|
||||
const hasDefaultPlugins = result.plugins?.some((p) => DEFAULT_PLUGINS.has(p.id as any));
|
||||
expect(hasDefaultPlugins).toBe(true);
|
||||
});
|
||||
|
||||
it('should migrate multilingual strategy languages to top-level language config', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [{ id: 'multilingual', config: { languages: ['es', 'fr'] } }],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Languages from multilingual strategy should be migrated to top-level language config
|
||||
expect(result.language).toEqual(['en', 'es', 'fr']);
|
||||
|
||||
// Multilingual strategy should be removed from strategies
|
||||
const hasMultilingual = result.strategies?.some(
|
||||
(s) => (typeof s === 'string' ? s : s.id) === 'multilingual',
|
||||
);
|
||||
expect(hasMultilingual).toBe(false);
|
||||
});
|
||||
|
||||
it('should merge multilingual strategy languages with existing language config', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [{ id: 'multilingual', config: { languages: ['de', 'it'] } }],
|
||||
language: 'ja',
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Should merge existing language with 'en' and multilingual languages
|
||||
expect(result.language).toEqual(['ja', 'en', 'de', 'it']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pluginOptions', () => {
|
||||
it('should not contain duplicate entries', () => {
|
||||
const duplicates = pluginOptions.filter((item, index) => pluginOptions.indexOf(item) !== index);
|
||||
expect(duplicates).toEqual([]);
|
||||
});
|
||||
|
||||
it('should contain unique values from all source arrays', () => {
|
||||
const manualSet = new Set([...COLLECTIONS, ...REDTEAM_ALL_PLUGINS, ...ALIASED_PLUGINS]);
|
||||
|
||||
expect(pluginOptions).toHaveLength(manualSet.size);
|
||||
expect(new Set(pluginOptions).size).toBe(pluginOptions.length);
|
||||
});
|
||||
|
||||
it('should be sorted alphabetically', () => {
|
||||
const sortedCopy = [...pluginOptions].sort();
|
||||
expect(pluginOptions).toEqual(sortedCopy);
|
||||
});
|
||||
|
||||
it('should contain all expected plugin types', () => {
|
||||
const hasCollectionItem = COLLECTIONS.some((item) => pluginOptions.includes(item));
|
||||
const hasRedteamPlugin = REDTEAM_ALL_PLUGINS.some((item) => pluginOptions.includes(item));
|
||||
const hasAliasedPlugin = ALIASED_PLUGINS.some((item) => pluginOptions.includes(item));
|
||||
|
||||
expect(hasCollectionItem).toBe(true);
|
||||
expect(hasRedteamPlugin).toBe(true);
|
||||
expect(hasAliasedPlugin).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle the specific bias duplicate case', () => {
|
||||
const biasCount = pluginOptions.filter((option) => option === 'bias').length;
|
||||
expect(biasCount).toBe(1);
|
||||
});
|
||||
|
||||
it('should deduplicate when source arrays have overlapping values', () => {
|
||||
// This test is specifically for the bias duplicate case
|
||||
const biasInCollections = COLLECTIONS.includes('bias' as any);
|
||||
const biasInAliased = ALIASED_PLUGINS.includes('bias' as any);
|
||||
|
||||
expect(biasInCollections).toBe(true);
|
||||
expect(biasInAliased).toBe(true);
|
||||
|
||||
const biasOccurrences = pluginOptions.filter((p) => p === 'bias');
|
||||
expect(biasOccurrences).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('redteam validators', () => {
|
||||
describe('strategyIdSchema', () => {
|
||||
describe('valid built-in strategies', () => {
|
||||
it('should accept standard built-in strategies', () => {
|
||||
const validStrategies = [
|
||||
'basic',
|
||||
'jailbreak',
|
||||
'crescendo',
|
||||
'goat',
|
||||
'multilingual',
|
||||
'base64',
|
||||
'hex',
|
||||
'rot13',
|
||||
];
|
||||
|
||||
validStrategies.forEach((strategy) => {
|
||||
expect(() => strategyIdSchema.parse(strategy)).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('custom strategy validation', () => {
|
||||
it('should accept simple custom strategy', () => {
|
||||
expect(() => strategyIdSchema.parse('custom')).not.toThrow();
|
||||
});
|
||||
|
||||
it('should accept custom strategy variants with compound IDs', () => {
|
||||
const customVariants = [
|
||||
'custom:aggressive',
|
||||
'custom:greeting-strategy',
|
||||
'custom:multi-word-variant',
|
||||
'custom:snake_case_variant',
|
||||
'custom:variant123',
|
||||
'custom:CamelCaseVariant',
|
||||
'custom:variant.with.dots',
|
||||
'custom:very-long-complex-variant-name-with-many-hyphens',
|
||||
];
|
||||
|
||||
customVariants.forEach((variant) => {
|
||||
expect(() => strategyIdSchema.parse(variant)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
it('should accept edge case custom strategy formats', () => {
|
||||
// These are actually valid custom strategies according to isCustomStrategy
|
||||
const validCustomStrategies = [
|
||||
'custom:', // Valid - empty variant
|
||||
'custom::', // Valid - double colon variant
|
||||
'custom:variant:', // Valid - trailing colon variant
|
||||
];
|
||||
|
||||
validCustomStrategies.forEach((strategy) => {
|
||||
expect(() => strategyIdSchema.parse(strategy)).not.toThrow();
|
||||
});
|
||||
|
||||
// Test invalid strategies that don't match any pattern
|
||||
const invalidStrategies = [
|
||||
':custom', // Invalid - doesn't start with 'custom'
|
||||
'invalid-strategy-123', // Invalid - not a built-in strategy
|
||||
];
|
||||
|
||||
invalidStrategies.forEach((strategy) => {
|
||||
expect(() => strategyIdSchema.parse(strategy)).toThrow(z.ZodError);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('file-based strategy validation', () => {
|
||||
it('should accept valid file:// strategy paths', () => {
|
||||
const validFilePaths = [
|
||||
'file://strategy.js',
|
||||
'file://strategy.ts',
|
||||
'file://./strategy.js',
|
||||
'file:///absolute/path/strategy.ts',
|
||||
'file://relative/path/strategy.js',
|
||||
];
|
||||
|
||||
validFilePaths.forEach((path) => {
|
||||
expect(() => strategyIdSchema.parse(path)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
it('should reject invalid file:// strategy paths', () => {
|
||||
const invalidFilePaths = [
|
||||
'file://strategy.txt',
|
||||
'file://strategy.py',
|
||||
'file://strategy',
|
||||
'strategy.js',
|
||||
'file:/strategy.js',
|
||||
];
|
||||
|
||||
invalidFilePaths.forEach((path) => {
|
||||
let error: z.ZodError | undefined;
|
||||
|
||||
expect(() => {
|
||||
try {
|
||||
strategyIdSchema.parse(path);
|
||||
} catch (e) {
|
||||
if (e instanceof z.ZodError) {
|
||||
error = e;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}).toThrow(z.ZodError);
|
||||
|
||||
expect(error).toBeDefined();
|
||||
const message = error?.issues[0].message || '';
|
||||
// Zod v4 may return "Invalid input" for union failures
|
||||
const hasValidMessage =
|
||||
message.includes('Custom strategies must start with file:// and end with .js or .ts') ||
|
||||
message.includes('Invalid input');
|
||||
expect(hasValidMessage).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('invalid strategy names', () => {
|
||||
it('should reject unknown strategy names', () => {
|
||||
// Test string strategies
|
||||
const invalidStringStrategies = [
|
||||
'unknown-strategy',
|
||||
'invalid_strategy',
|
||||
'not-a-strategy',
|
||||
'fakeStrategy',
|
||||
];
|
||||
|
||||
invalidStringStrategies.forEach((strategy) => {
|
||||
let error: z.ZodError | undefined;
|
||||
|
||||
expect(() => {
|
||||
try {
|
||||
strategyIdSchema.parse(strategy);
|
||||
} catch (e) {
|
||||
if (e instanceof z.ZodError) {
|
||||
error = e;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}).toThrow(z.ZodError);
|
||||
|
||||
expect(error).toBeDefined();
|
||||
const message = error?.issues[0].message || '';
|
||||
const validMessages = [
|
||||
'Custom strategies must start with file:// and end with .js or .ts',
|
||||
'Strategy must be one of the built-in strategies:',
|
||||
'Invalid enum value',
|
||||
'Invalid input', // Zod v4 generic union error
|
||||
];
|
||||
const hasValidMessage = validMessages.some((msg) => message.includes(msg));
|
||||
expect(hasValidMessage).toBe(true);
|
||||
});
|
||||
|
||||
// Test null/undefined/empty separately
|
||||
expect(() => strategyIdSchema.parse(null)).toThrow(z.ZodError);
|
||||
expect(() => strategyIdSchema.parse(undefined)).toThrow(z.ZodError);
|
||||
expect(() => strategyIdSchema.parse('')).toThrow(z.ZodError);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('RedteamStrategySchema', () => {
|
||||
describe('string format strategies', () => {
|
||||
it('should accept valid strategy strings', () => {
|
||||
const validStrategies = [
|
||||
'basic',
|
||||
'custom',
|
||||
'custom:variant',
|
||||
'jailbreak',
|
||||
'crescendo',
|
||||
'file://strategy.js',
|
||||
];
|
||||
|
||||
validStrategies.forEach((strategy) => {
|
||||
expect(() => RedteamStrategySchema.parse(strategy)).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('object format strategies', () => {
|
||||
it('should accept strategy objects with valid IDs', () => {
|
||||
const validStrategyObjects = [
|
||||
{ id: 'basic' },
|
||||
{ id: 'custom' },
|
||||
{ id: 'custom:aggressive' },
|
||||
{ id: 'jailbreak', config: { enabled: true } },
|
||||
{ id: 'custom:greeting', config: { strategyText: 'Be polite', stateful: true } },
|
||||
{ id: 'multilingual', config: { languages: ['es', 'fr'] } },
|
||||
{ id: 'file://custom.js', config: { param: 'value' } },
|
||||
];
|
||||
|
||||
validStrategyObjects.forEach((strategy) => {
|
||||
expect(() => RedteamStrategySchema.parse(strategy)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
it('should accept custom strategy objects with complex configurations', () => {
|
||||
const customStrategyWithConfig = {
|
||||
id: 'custom:complex-variant',
|
||||
config: {
|
||||
strategyText:
|
||||
'If current round is 0, generatedQuestion should be just "hi" by itself...',
|
||||
stateful: true,
|
||||
temperature: 0.8,
|
||||
maxTokens: 100,
|
||||
customParam: true,
|
||||
nestedConfig: {
|
||||
subParam: 'value',
|
||||
subArray: [1, 2, 3],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(() => RedteamStrategySchema.parse(customStrategyWithConfig)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should reject strategy objects with invalid IDs', () => {
|
||||
// Test objects with invalid IDs
|
||||
const invalidIdObjects = [
|
||||
{ id: 'custom-invalid' },
|
||||
{ id: 'notcustom:variant' },
|
||||
{ id: 'file://invalid.txt' },
|
||||
{ id: '' },
|
||||
];
|
||||
|
||||
invalidIdObjects.forEach((strategy) => {
|
||||
let error: z.ZodError | undefined;
|
||||
|
||||
expect(() => {
|
||||
try {
|
||||
RedteamStrategySchema.parse(strategy);
|
||||
} catch (e) {
|
||||
if (e instanceof z.ZodError) {
|
||||
error = e;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}).toThrow(z.ZodError);
|
||||
|
||||
expect(error).toBeDefined();
|
||||
const message = error?.issues[0].message || '';
|
||||
const validMessages = [
|
||||
'Custom strategies must start with file:// and end with .js or .ts',
|
||||
'Strategy must be one of the built-in strategies:',
|
||||
'Invalid enum value',
|
||||
'Invalid input', // Zod v4 generic union error
|
||||
];
|
||||
const hasValidMessage = validMessages.some((msg) => message.includes(msg));
|
||||
expect(hasValidMessage).toBe(true);
|
||||
});
|
||||
|
||||
// Test object without id separately
|
||||
expect(() => RedteamStrategySchema.parse({})).toThrow(z.ZodError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
it('should handle strategy objects with empty config', () => {
|
||||
const strategyWithEmptyConfig = { id: 'custom:variant', config: {} };
|
||||
expect(() => RedteamStrategySchema.parse(strategyWithEmptyConfig)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle strategy objects with undefined config', () => {
|
||||
const strategyWithUndefinedConfig = { id: 'custom:variant', config: undefined };
|
||||
expect(() => RedteamStrategySchema.parse(strategyWithUndefinedConfig)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle strategy objects without config property', () => {
|
||||
const strategyWithoutConfig = { id: 'custom:variant' };
|
||||
expect(() => RedteamStrategySchema.parse(strategyWithoutConfig)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('error messages', () => {
|
||||
it('should provide helpful error message for invalid file strategy', () => {
|
||||
let error: z.ZodError | undefined;
|
||||
|
||||
expect(() => {
|
||||
try {
|
||||
strategyIdSchema.parse('file://strategy.txt');
|
||||
} catch (e) {
|
||||
if (e instanceof z.ZodError) {
|
||||
error = e;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}).toThrow(z.ZodError);
|
||||
|
||||
expect(error).toBeDefined();
|
||||
const message = error?.issues[0].message || '';
|
||||
// Zod v4 may return "Invalid input" for union failures or the specific message
|
||||
const hasValidMessage =
|
||||
message.includes('Custom strategies must start with file:// and end with .js or .ts') ||
|
||||
message.includes('Invalid input');
|
||||
expect(hasValidMessage).toBe(true);
|
||||
});
|
||||
|
||||
it('should provide helpful error message for completely invalid strategy', () => {
|
||||
let error: z.ZodError | undefined;
|
||||
|
||||
expect(() => {
|
||||
try {
|
||||
strategyIdSchema.parse('totally-invalid-strategy');
|
||||
} catch (e) {
|
||||
if (e instanceof z.ZodError) {
|
||||
error = e;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}).toThrow(z.ZodError);
|
||||
|
||||
expect(error).toBeDefined();
|
||||
const message = error?.issues[0].message || '';
|
||||
const validMessages = [
|
||||
'Custom strategies must start with file:// and end with .js or .ts',
|
||||
'Strategy must be one of the built-in strategies:',
|
||||
'Invalid enum value',
|
||||
'Invalid input', // Zod v4 generic union error
|
||||
];
|
||||
const hasValidMessage = validMessages.some((msg) => message.includes(msg));
|
||||
expect(hasValidMessage).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('integration with actual use cases', () => {
|
||||
it('should validate realistic custom strategy configurations', () => {
|
||||
const realisticConfigurations = [
|
||||
'custom:greeting-strategy',
|
||||
'custom:default-strategy',
|
||||
{
|
||||
id: 'custom:greeting-strategy',
|
||||
config: {
|
||||
stateful: true,
|
||||
strategyText:
|
||||
"If current round is 0, generatedQuestion should be just 'Hello can you help me?' by itself...",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'custom:default-strategy',
|
||||
config: {
|
||||
stateful: true,
|
||||
strategyText:
|
||||
"If current round is 0, generatedQuestion should be just 'hi' by itself...",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
realisticConfigurations.forEach((config) => {
|
||||
expect(() => RedteamStrategySchema.parse(config)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
it('should validate mixed strategy arrays like those used in real configurations', () => {
|
||||
const mixedStrategies = [
|
||||
'basic',
|
||||
{ id: 'jailbreak', config: { enabled: true } },
|
||||
'custom:aggressive-variant',
|
||||
{
|
||||
id: 'custom:polite-variant',
|
||||
config: {
|
||||
strategyText: 'Be very polite and formal',
|
||||
stateful: false,
|
||||
},
|
||||
},
|
||||
'crescendo',
|
||||
'file://./my-custom-strategy.js',
|
||||
];
|
||||
|
||||
mixedStrategies.forEach((strategy) => {
|
||||
expect(() => RedteamStrategySchema.parse(strategy)).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('layer strategy deduplication', () => {
|
||||
it('should keep multiple layer strategies with different labels', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [
|
||||
{ id: 'layer', config: { label: 'hydra-audio', steps: ['jailbreak:hydra', 'audio'] } },
|
||||
{ id: 'layer', config: { label: 'crescendo-audio', steps: ['crescendo', 'audio'] } },
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Both layer strategies should be kept (different labels)
|
||||
const layerStrategies = result.strategies?.filter(
|
||||
(s) => typeof s !== 'string' && s.id === 'layer',
|
||||
);
|
||||
expect(layerStrategies).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should keep multiple layer strategies with different steps', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [
|
||||
{ id: 'layer', config: { steps: ['jailbreak:hydra', 'audio'] } },
|
||||
{ id: 'layer', config: { steps: ['crescendo', 'image'] } },
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Both layer strategies should be kept (different steps)
|
||||
const layerStrategies = result.strategies?.filter(
|
||||
(s) => typeof s !== 'string' && s.id === 'layer',
|
||||
);
|
||||
expect(layerStrategies).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should deduplicate identical layer strategies', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [
|
||||
{ id: 'layer', config: { label: 'same-label', steps: ['jailbreak:hydra', 'audio'] } },
|
||||
{ id: 'layer', config: { label: 'same-label', steps: ['jailbreak:hydra', 'audio'] } },
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Duplicate layer strategies should be deduplicated
|
||||
const layerStrategies = result.strategies?.filter(
|
||||
(s) => typeof s !== 'string' && s.id === 'layer',
|
||||
);
|
||||
expect(layerStrategies).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should use label for deduplication key when provided', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [
|
||||
{ id: 'layer', config: { label: 'unique-label', steps: ['jailbreak:hydra', 'audio'] } },
|
||||
{ id: 'layer', config: { label: 'unique-label', steps: ['crescendo', 'image'] } }, // Same label, different steps
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Should only keep one (label is the key, so second overwrites first)
|
||||
const layerStrategies = result.strategies?.filter(
|
||||
(s) => typeof s !== 'string' && s.id === 'layer',
|
||||
);
|
||||
expect(layerStrategies).toHaveLength(1);
|
||||
// The last one wins
|
||||
expect(layerStrategies?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
config: expect.objectContaining({ steps: ['crescendo', 'image'] }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle layer strategies without label or steps', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [{ id: 'layer', config: {} }],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
const layerStrategies = result.strategies?.filter(
|
||||
(s) => typeof s !== 'string' && s.id === 'layer',
|
||||
);
|
||||
expect(layerStrategies).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should handle mixed layer and non-layer strategies', () => {
|
||||
const config = {
|
||||
plugins: ['default'],
|
||||
strategies: [
|
||||
'basic',
|
||||
{ id: 'layer', config: { label: 'audio-attack', steps: ['jailbreak:hydra', 'audio'] } },
|
||||
'jailbreak',
|
||||
{ id: 'layer', config: { label: 'image-attack', steps: ['crescendo', 'image'] } },
|
||||
],
|
||||
};
|
||||
|
||||
const result = RedteamConfigSchema.parse(config);
|
||||
|
||||
// Should have all strategies
|
||||
expect(result.strategies?.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
// Layer strategies should both be present
|
||||
const layerStrategies = result.strategies?.filter(
|
||||
(s) => typeof s !== 'string' && s.id === 'layer',
|
||||
);
|
||||
expect(layerStrategies).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error message quality', () => {
|
||||
it('should provide clear error messages for common mistakes', () => {
|
||||
// Test various common mistakes and ensure error messages are helpful
|
||||
// Note: Zod v4 may return "Invalid input" for union failures
|
||||
const testCases = [
|
||||
{
|
||||
config: { plugins: ['non-existent-plugin'] },
|
||||
expectedError: /Custom plugins must start with file:\/\/|Invalid input/,
|
||||
description: 'non-existent plugin name',
|
||||
},
|
||||
{
|
||||
config: { strategies: ['file://strategy.json'] },
|
||||
expectedError:
|
||||
/Custom strategies must start with file:\/\/ and end with .js or .ts|Invalid input/,
|
||||
description: 'wrong file extension for strategy',
|
||||
},
|
||||
{
|
||||
config: { strategies: ['invalid-strategy-name'] },
|
||||
expectedError:
|
||||
/Custom strategies must start with file:\/\/ and end with .js or .ts|Invalid input/,
|
||||
description: 'invalid strategy name',
|
||||
},
|
||||
{
|
||||
config: { plugins: [{ id: 'default' }], numTests: 'many' as any },
|
||||
expectedError: /Expected number|Invalid type|Invalid input|expected number/i,
|
||||
description: 'string instead of number for numTests',
|
||||
},
|
||||
{
|
||||
config: { plugins: ['default'], delay: 'slow' as any },
|
||||
expectedError: /Expected number|Invalid type|Invalid input|expected number/i,
|
||||
description: 'string instead of number for delay',
|
||||
},
|
||||
];
|
||||
|
||||
testCases.forEach(({ config, expectedError }) => {
|
||||
let errorCaught = false;
|
||||
let zodError: z.ZodError | null = null;
|
||||
|
||||
try {
|
||||
RedteamConfigSchema.parse(config);
|
||||
} catch (error) {
|
||||
errorCaught = true;
|
||||
if (error instanceof z.ZodError) {
|
||||
zodError = error;
|
||||
}
|
||||
}
|
||||
|
||||
expect(errorCaught).toBe(true);
|
||||
expect(zodError).not.toBeNull();
|
||||
|
||||
const message = zodError!.issues[0]?.message || '';
|
||||
|
||||
// Always perform the assertion, the expected pattern handles both string and regex
|
||||
const matches =
|
||||
typeof expectedError === 'string'
|
||||
? message.includes(expectedError)
|
||||
: expectedError.test(message);
|
||||
|
||||
expect(matches).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { NunjucksFilterMapSchema } from '../../src/validators/shared';
|
||||
|
||||
describe('NunjucksFilterMapSchema', () => {
|
||||
it('should accept empty object', () => {
|
||||
const result = NunjucksFilterMapSchema.safeParse({});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toEqual({});
|
||||
});
|
||||
|
||||
it('should accept valid filter map with functions', () => {
|
||||
const input = {
|
||||
uppercase: (val: unknown) => String(val).toUpperCase(),
|
||||
lowercase: (val: unknown) => String(val).toLowerCase(),
|
||||
};
|
||||
const result = NunjucksFilterMapSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveProperty('uppercase');
|
||||
expect(result.data).toHaveProperty('lowercase');
|
||||
});
|
||||
|
||||
it('should accept single filter', () => {
|
||||
const input = {
|
||||
trim: (val: unknown) => String(val).trim(),
|
||||
};
|
||||
const result = NunjucksFilterMapSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data).toHaveProperty('trim');
|
||||
});
|
||||
|
||||
it('should reject non-function values', () => {
|
||||
const result = NunjucksFilterMapSchema.safeParse({
|
||||
notAFunction: 'string-value',
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject number values', () => {
|
||||
const result = NunjucksFilterMapSchema.safeParse({
|
||||
badFilter: 42,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject null values', () => {
|
||||
const result = NunjucksFilterMapSchema.safeParse({
|
||||
nullFilter: null,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject boolean values', () => {
|
||||
const result = NunjucksFilterMapSchema.safeParse({
|
||||
boolFilter: true,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should accept functions with multiple arguments', () => {
|
||||
const input = {
|
||||
replace: (val: unknown, search: unknown, replacement: unknown) =>
|
||||
String(val).replace(String(search), String(replacement)),
|
||||
};
|
||||
const result = NunjucksFilterMapSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should reject non-object input', () => {
|
||||
expect(NunjucksFilterMapSchema.safeParse('not-an-object').success).toBe(false);
|
||||
expect(NunjucksFilterMapSchema.safeParse(42).success).toBe(false);
|
||||
expect(NunjucksFilterMapSchema.safeParse(null).success).toBe(false);
|
||||
expect(NunjucksFilterMapSchema.safeParse(undefined).success).toBe(false);
|
||||
});
|
||||
|
||||
it('should reject array input', () => {
|
||||
const result = NunjucksFilterMapSchema.safeParse([() => 'test']);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should accept mix of valid function filters', () => {
|
||||
const input = {
|
||||
first: () => 'a',
|
||||
second: (_a: unknown, _b: unknown) => 'b',
|
||||
third: (..._args: unknown[]) => 'c',
|
||||
};
|
||||
const result = NunjucksFilterMapSchema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
expect(Object.keys(result.data!)).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,259 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../src/logger', () => ({
|
||||
default: {
|
||||
debug: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
import {
|
||||
determineEffectiveSessionSource,
|
||||
formatConfigBody,
|
||||
formatConfigHeaders,
|
||||
validateSessionConfig,
|
||||
} from '../../src/validators/util';
|
||||
|
||||
import type { ApiProvider } from '../../src/types/providers';
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('formatConfigBody', () => {
|
||||
it('should return "None configured" when body is falsy', () => {
|
||||
expect(formatConfigBody({ body: undefined })).toBe('None configured');
|
||||
expect(formatConfigBody({ body: null })).toBe('None configured');
|
||||
expect(formatConfigBody({ body: '' })).toBe('None configured');
|
||||
});
|
||||
|
||||
it('should pretty-print valid JSON string body', () => {
|
||||
const body = '{"key":"value","nested":{"a":1}}';
|
||||
const result = formatConfigBody({ body });
|
||||
expect(result).toContain('"key": "value"');
|
||||
expect(result).toContain('"nested"');
|
||||
// Should be indented
|
||||
expect(result).toContain(' ');
|
||||
});
|
||||
|
||||
it('should indent non-JSON string body', () => {
|
||||
const body = 'plain text body';
|
||||
const result = formatConfigBody({ body });
|
||||
expect(result).toBe('\n plain text body');
|
||||
});
|
||||
|
||||
it('should stringify and indent object body', () => {
|
||||
const body = { key: 'value', number: 42 };
|
||||
const result = formatConfigBody({ body });
|
||||
expect(result).toContain('"key": "value"');
|
||||
expect(result).toContain('"number": 42');
|
||||
expect(result).toContain(' ');
|
||||
});
|
||||
|
||||
it('should handle nested objects', () => {
|
||||
const body = { outer: { inner: 'deep' } };
|
||||
const result = formatConfigBody({ body });
|
||||
expect(result).toContain('"outer"');
|
||||
expect(result).toContain('"inner": "deep"');
|
||||
});
|
||||
|
||||
it('should handle array body', () => {
|
||||
const body = [1, 2, 3];
|
||||
const result = formatConfigBody({ body });
|
||||
expect(result).toContain('1');
|
||||
expect(result).toContain('2');
|
||||
expect(result).toContain('3');
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatConfigHeaders', () => {
|
||||
it('should return "None configured" when headers is undefined', () => {
|
||||
expect(formatConfigHeaders({ headers: undefined })).toBe('None configured');
|
||||
});
|
||||
|
||||
it('should format headers as key-value pairs', () => {
|
||||
const headers = {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer token123',
|
||||
};
|
||||
const result = formatConfigHeaders({ headers });
|
||||
expect(result).toContain('Content-Type: application/json');
|
||||
expect(result).toContain('Authorization: Bearer token123');
|
||||
});
|
||||
|
||||
it('should handle single header', () => {
|
||||
const result = formatConfigHeaders({ headers: { Accept: 'text/plain' } });
|
||||
expect(result).toContain('Accept: text/plain');
|
||||
});
|
||||
|
||||
it('should handle empty headers object', () => {
|
||||
const result = formatConfigHeaders({ headers: {} });
|
||||
// Empty object should produce a newline with no entries
|
||||
expect(result).toBe('\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateSessionConfig', () => {
|
||||
it('should warn when sessionId template is missing from config', () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
|
||||
// The function uses the logger module, so we just verify it doesn't throw
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { url: 'https://example.com/api' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
// The function uses the logger module, so we just verify it doesn't throw
|
||||
validateSessionConfig({
|
||||
provider,
|
||||
sessionSource: 'client',
|
||||
});
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('should not warn when sessionId template is present in config', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: {
|
||||
url: 'https://example.com/api',
|
||||
headers: { 'X-Session': '{{sessionId}}' },
|
||||
},
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
// Should not throw
|
||||
expect(() =>
|
||||
validateSessionConfig({
|
||||
provider,
|
||||
sessionSource: 'client',
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle server session source without session parser', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { url: 'https://example.com/api', body: '{{sessionId}}' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
// Should not throw even with missing session parser
|
||||
expect(() =>
|
||||
validateSessionConfig({
|
||||
provider,
|
||||
sessionSource: 'server',
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle server session source with session parser', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { url: 'https://example.com/api', body: '{{sessionId}}' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
expect(() =>
|
||||
validateSessionConfig({
|
||||
provider,
|
||||
sessionSource: 'server',
|
||||
sessionConfig: { sessionParser: 'response.headers.x-session-id' },
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('should handle provider with no config', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: {},
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
expect(() =>
|
||||
validateSessionConfig({
|
||||
provider,
|
||||
sessionSource: 'client',
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('determineEffectiveSessionSource', () => {
|
||||
it('should use sessionConfig.sessionSource when provided', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { sessionSource: 'client' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
const result = determineEffectiveSessionSource({
|
||||
provider,
|
||||
sessionConfig: { sessionSource: 'server' },
|
||||
});
|
||||
expect(result).toBe('server');
|
||||
});
|
||||
|
||||
it('should use provider.config.sessionSource when sessionConfig is not provided', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { sessionSource: 'server' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
const result = determineEffectiveSessionSource({ provider });
|
||||
expect(result).toBe('server');
|
||||
});
|
||||
|
||||
it('should return "server" when provider has sessionParser', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { sessionParser: 'response.id' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
const result = determineEffectiveSessionSource({ provider });
|
||||
expect(result).toBe('server');
|
||||
});
|
||||
|
||||
it('should default to "client" when no session config exists', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: {},
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
const result = determineEffectiveSessionSource({ provider });
|
||||
expect(result).toBe('client');
|
||||
});
|
||||
|
||||
it('should prioritize sessionConfig over provider config', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: { sessionSource: 'client', sessionParser: 'response.id' },
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
const result = determineEffectiveSessionSource({
|
||||
provider,
|
||||
sessionConfig: { sessionSource: 'server' },
|
||||
});
|
||||
expect(result).toBe('server');
|
||||
});
|
||||
|
||||
it('should handle undefined sessionConfig', () => {
|
||||
const provider = {
|
||||
id: () => 'test-provider',
|
||||
config: {},
|
||||
callApi: vi.fn(),
|
||||
} as unknown as ApiProvider;
|
||||
|
||||
const result = determineEffectiveSessionSource({
|
||||
provider,
|
||||
sessionConfig: undefined,
|
||||
});
|
||||
expect(result).toBe('client');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user