chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
+141
View File
@@ -0,0 +1,141 @@
import { describe, expect, it } from 'vitest';
import { PromptConfigSchema, PromptSchema } from '../../src/validators/prompts';
describe('PromptConfigSchema', () => {
it('should accept empty object', () => {
const result = PromptConfigSchema.safeParse({});
expect(result.success).toBe(true);
expect(result.data).toEqual({});
});
it('should accept valid prefix and suffix', () => {
const input = { prefix: 'You are a helpful assistant.', suffix: 'Please respond.' };
const result = PromptConfigSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toEqual(input);
});
it('should accept only prefix', () => {
const input = { prefix: 'System: ' };
const result = PromptConfigSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toEqual(input);
});
it('should accept only suffix', () => {
const input = { suffix: '\nEnd.' };
const result = PromptConfigSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toEqual(input);
});
it('should accept empty string values', () => {
const input = { prefix: '', suffix: '' };
const result = PromptConfigSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toEqual(input);
});
it('should reject non-string prefix', () => {
const result = PromptConfigSchema.safeParse({ prefix: 123 });
expect(result.success).toBe(false);
});
it('should reject non-string suffix', () => {
const result = PromptConfigSchema.safeParse({ suffix: true });
expect(result.success).toBe(false);
});
});
describe('PromptSchema', () => {
it('should accept valid prompt with required fields', () => {
const input = { raw: 'Hello {{name}}', label: 'Greeting prompt' };
const result = PromptSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toMatchObject(input);
});
it('should accept prompt with all fields', () => {
const mockFn = async () => 'response';
const input = {
id: 'prompt-1',
raw: 'Tell me about {{topic}}',
display: 'Topic prompt (deprecated)',
label: 'Topic prompt',
function: mockFn,
config: { temperature: 0.7 },
};
const result = PromptSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toHaveProperty('id', 'prompt-1');
expect(result.data).toHaveProperty('raw', 'Tell me about {{topic}}');
expect(result.data).toHaveProperty('label', 'Topic prompt');
expect(result.data).toHaveProperty('function');
expect(result.data).toHaveProperty('config');
});
it('should reject missing raw field', () => {
const result = PromptSchema.safeParse({ label: 'No raw' });
expect(result.success).toBe(false);
});
it('should reject missing label field', () => {
const result = PromptSchema.safeParse({ raw: 'Hello' });
expect(result.success).toBe(false);
});
it('should reject empty object', () => {
const result = PromptSchema.safeParse({});
expect(result.success).toBe(false);
});
it('should accept prompt with optional id', () => {
const result = PromptSchema.safeParse({
id: 'test-id',
raw: 'prompt text',
label: 'test label',
});
expect(result.success).toBe(true);
expect(result.data).toHaveProperty('id', 'test-id');
});
it('should accept prompt without optional fields', () => {
const result = PromptSchema.safeParse({
raw: 'minimal prompt',
label: 'minimal',
});
expect(result.success).toBe(true);
expect(result.data).not.toHaveProperty('id');
expect(result.data).not.toHaveProperty('display');
expect(result.data).not.toHaveProperty('function');
});
it('should reject non-function for function field', () => {
const result = PromptSchema.safeParse({
raw: 'test',
label: 'test',
function: 'not-a-function',
});
expect(result.success).toBe(false);
});
it('should accept any config value', () => {
const result = PromptSchema.safeParse({
raw: 'test',
label: 'test',
config: { nested: { deep: true }, array: [1, 2, 3] },
});
expect(result.success).toBe(true);
expect(result.data?.config).toEqual({ nested: { deep: true }, array: [1, 2, 3] });
});
it('should reject non-string raw value', () => {
const result = PromptSchema.safeParse({ raw: 42, label: 'test' });
expect(result.success).toBe(false);
});
it('should reject non-string label value', () => {
const result = PromptSchema.safeParse({ raw: 'test', label: 123 });
expect(result.success).toBe(false);
});
});
+85
View File
@@ -0,0 +1,85 @@
import { describe, expect, it } from 'vitest';
import { ProviderOptionsSchema, ProviderSchema } from '../../src/validators/providers';
import { createMockProvider } from '../factories/provider';
describe('ProviderOptionsSchema', () => {
it('should filter unknown keys without erroring', () => {
const input = {
id: 'test-provider',
label: 'Test Provider',
unknownField: 'this should be filtered',
anotherUnknown: 123,
};
const result = ProviderOptionsSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toHaveProperty('id', 'test-provider');
expect(result.data).toHaveProperty('label', 'Test Provider');
expect(result.data).not.toHaveProperty('unknownField');
expect(result.data).not.toHaveProperty('anotherUnknown');
});
it('should accept valid provider options', () => {
const input = {
id: 'test-provider',
label: 'Test Provider',
config: { temperature: 0.7 },
prompts: ['prompt1', 'prompt2'],
transform: 'output.toLowerCase()',
delay: 1000,
};
const result = ProviderOptionsSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toEqual(input);
});
it('should accept empty object', () => {
const result = ProviderOptionsSchema.safeParse({});
expect(result.success).toBe(true);
expect(result.data).toEqual({});
});
});
describe('ProviderSchema union', () => {
it('should match ApiProviderSchema before ProviderOptionsSchema when callApi is present', () => {
const input = createMockProvider({
id: 'custom-provider',
label: 'Custom Provider',
});
const result = ProviderSchema.safeParse(input);
expect(result.success).toBe(true);
// callApi should be preserved because ApiProviderSchema matches first
expect(result.data).toHaveProperty('callApi');
expect(result.data).toHaveProperty('id');
expect(result.data).toHaveProperty('label', 'Custom Provider');
});
it('should match ProviderOptionsSchema when no callApi function', () => {
const input = {
id: 'test-provider',
label: 'Test Provider',
unknownField: 'should be filtered',
};
const result = ProviderSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toHaveProperty('id', 'test-provider');
expect(result.data).toHaveProperty('label', 'Test Provider');
// unknownField should be filtered by ProviderOptionsSchema
expect(result.data).not.toHaveProperty('unknownField');
});
it('should accept string provider', () => {
const result = ProviderSchema.safeParse('openai:gpt-4');
expect(result.success).toBe(true);
expect(result.data).toBe('openai:gpt-4');
});
});
+976
View File
@@ -0,0 +1,976 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { z } from 'zod';
import {
ALIASED_PLUGIN_MAPPINGS,
ALIASED_PLUGINS,
COLLECTIONS,
DEFAULT_NUM_TESTS_PER_PLUGIN,
DEFAULT_PLUGINS,
FOUNDATION_PLUGINS,
HARM_PLUGINS,
PII_PLUGINS,
ALL_PLUGINS as REDTEAM_ALL_PLUGINS,
Severity,
TEEN_SAFETY_PLUGINS,
} from '../../src/redteam/constants';
import {
pluginOptions,
RedteamConfigSchema,
RedteamStrategySchema,
strategyIdSchema,
} from '../../src/validators/redteam';
beforeEach(() => {
vi.spyOn(console, 'debug').mockImplementation(() => {});
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('RedteamConfigSchema', () => {
it('should validate basic config', () => {
const config = {
plugins: ['default'],
strategies: ['default'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins).toBeDefined();
expect(result.strategies).toBeDefined();
});
it('should expand default plugins', () => {
const config = {
plugins: ['default'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.length).toBeGreaterThan(0);
expect(result.plugins?.every((p: any) => Array.from(DEFAULT_PLUGINS).includes(p.id))).toBe(
true,
);
});
it('should expand foundation plugins', () => {
const config = {
plugins: ['foundation'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.length).toBeGreaterThan(0);
expect(result.plugins?.every((p: any) => Array.from(FOUNDATION_PLUGINS).includes(p.id))).toBe(
true,
);
});
it('should expand harmful plugins', () => {
const config = {
plugins: ['harmful'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.length).toBeGreaterThan(0);
expect(result.plugins?.every((p: any) => Object.keys(HARM_PLUGINS).includes(p.id))).toBe(true);
});
it('should expand PII plugins', () => {
const config = {
plugins: ['pii'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.length).toBeGreaterThan(0);
expect(result.plugins?.every((p: any) => Array.from(PII_PLUGINS).includes(p.id))).toBe(true);
});
it('should expand teen safety plugins', () => {
const config = {
plugins: ['teen-safety'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.map((plugin) => plugin.id).sort()).toEqual(
[...TEEN_SAFETY_PLUGINS].sort(),
);
});
it('should handle plugin with config and severity', () => {
const config = {
plugins: [
{
id: 'file://test-plugin.js',
config: { key: 'value' },
severity: Severity.High,
numTests: DEFAULT_NUM_TESTS_PER_PLUGIN,
},
],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.[0]).toEqual({
id: 'file://test-plugin.js',
config: { key: 'value' },
severity: Severity.High,
numTests: DEFAULT_NUM_TESTS_PER_PLUGIN,
});
});
it('should handle aliased plugins', () => {
const config = {
plugins: [Object.keys(ALIASED_PLUGIN_MAPPINGS)[0]],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.length).toBeGreaterThan(0);
});
it('should validate custom file:// plugins', () => {
const config = {
plugins: ['file:///path/to/plugin.js'],
};
expect(() => RedteamConfigSchema.parse(config)).not.toThrow();
});
it('should reject invalid plugin names', () => {
const config = {
plugins: ['invalid-plugin-name'],
};
expect(() => RedteamConfigSchema.parse(config)).toThrow(/Invalid plugin id/);
});
it('should handle numTests override', () => {
const config = {
numTests: 5,
plugins: ['file://test-plugin.js'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.[0].numTests).toBe(5);
});
it('should inherit top-level numTests for object plugins that omit numTests', () => {
const config = {
numTests: 17,
plugins: [{ id: 'file://test-plugin.js' }],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.[0].numTests).toBe(17);
});
it('should use default numTests when not specified', () => {
const config = {
plugins: [
{
id: 'file://test-plugin.js',
numTests: DEFAULT_NUM_TESTS_PER_PLUGIN,
},
],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.[0].numTests).toBe(DEFAULT_NUM_TESTS_PER_PLUGIN);
});
it('should deduplicate plugins', () => {
const config = {
plugins: ['file://test-plugin.js', 'file://test-plugin.js'],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins).toHaveLength(1);
});
it('should handle collection plugins', () => {
const config = {
plugins: COLLECTIONS,
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.length).toBeGreaterThan(0);
});
it('should validate RedteamPluginObjectSchema', () => {
const validPlugin = {
id: 'file://test-plugin.js',
numTests: 5,
config: { key: 'value' },
severity: Severity.High,
};
const config = {
plugins: [validPlugin],
};
expect(() => RedteamConfigSchema.parse(config)).not.toThrow();
});
it('should handle plugin with severity but no config', () => {
const config = {
plugins: [
{
id: 'file://test-plugin.js',
severity: Severity.Low,
},
],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.[0].severity).toBe(Severity.Low);
expect(result.plugins?.[0].config).toBeUndefined();
});
it('should preserve severity when expanding collections', () => {
const config = {
plugins: [
{
id: 'foundation',
severity: Severity.Critical,
},
],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins?.every((p) => p.severity === Severity.Critical)).toBe(true);
});
it('should handle strategy with config', () => {
const config = {
plugins: ['default'],
strategies: [{ id: 'default', config: { key: 'value' } }],
};
const result = RedteamConfigSchema.parse(config);
expect(result.strategies?.[0]).toEqual(expect.objectContaining({ config: { key: 'value' } }));
});
it('should handle custom file:// strategies', () => {
const config = {
plugins: ['default'],
strategies: ['file:///path/to/strategy.js'],
};
expect(() => RedteamConfigSchema.parse(config)).not.toThrow();
});
it('should handle empty plugins array', () => {
const config = {
plugins: [],
};
const result = RedteamConfigSchema.parse(config);
expect(result.plugins).toEqual([]);
});
it('should handle config with all optional fields', () => {
const config = {
plugins: ['default'],
strategies: ['basic'],
numTests: 10,
language: 'en',
injectVar: 'prompt',
purpose: 'Testing LLM robustness',
entities: ['Company A', 'Product B'],
maxConcurrency: 5,
delay: 1000,
provider: 'openai:gpt-4',
};
const result = RedteamConfigSchema.parse(config);
expect(result.numTests).toBe(10);
expect(result.language).toBe('en');
expect(result.injectVar).toBe('prompt');
expect(result.purpose).toBe('Testing LLM robustness');
expect(result.entities).toEqual(['Company A', 'Product B']);
expect(result.delay).toBe(1000);
expect(result.provider).toBe('openai:gpt-4');
});
it('should reject invalid numTests values', () => {
const invalidConfigs = [
{ plugins: ['default'], numTests: 0 }, // Zero
{ plugins: ['default'], numTests: -5 }, // Negative
{ plugins: ['default'], numTests: 1.5 }, // Decimal
{ plugins: ['default'], numTests: 'five' as any }, // String
];
invalidConfigs.forEach((config) => {
expect(() => RedteamConfigSchema.parse(config)).toThrow(z.ZodError);
});
});
it('should reject invalid delay values', () => {
const invalidConfigs = [
{ plugins: ['default'], delay: -100 }, // Negative delay
{ plugins: ['default'], delay: 'slow' as any }, // String
];
invalidConfigs.forEach((config) => {
expect(() => RedteamConfigSchema.parse(config)).toThrow(z.ZodError);
});
});
it('should properly transform and deduplicate complex plugin configurations', () => {
const config = {
plugins: [
'default',
'default', // Duplicate
{ id: 'contracts', numTests: 10 },
{ id: 'contracts', numTests: 10 }, // Exact duplicate
{ id: 'contracts', numTests: 5 }, // Different numTests - this one will be kept
'file://custom1.js',
'file://custom1.js', // Duplicate file
],
};
const result = RedteamConfigSchema.parse(config);
// The transform function deduplicates based on key: `${id}:${JSON.stringify(config)}:${severity || ''}`
// So contracts with different numTests are different plugins
const contractsPlugins = result.plugins?.filter((p) => p.id === 'contracts') || [];
expect(contractsPlugins.length).toBeGreaterThan(0);
// File plugins are deduplicated
const customPlugins = result.plugins?.filter((p) => p.id === 'file://custom1.js') || [];
expect(customPlugins).toHaveLength(1); // Deduplicated
// Default plugins should be expanded
const hasDefaultPlugins = result.plugins?.some((p) => DEFAULT_PLUGINS.has(p.id as any));
expect(hasDefaultPlugins).toBe(true);
});
it('should migrate multilingual strategy languages to top-level language config', () => {
const config = {
plugins: ['default'],
strategies: [{ id: 'multilingual', config: { languages: ['es', 'fr'] } }],
};
const result = RedteamConfigSchema.parse(config);
// Languages from multilingual strategy should be migrated to top-level language config
expect(result.language).toEqual(['en', 'es', 'fr']);
// Multilingual strategy should be removed from strategies
const hasMultilingual = result.strategies?.some(
(s) => (typeof s === 'string' ? s : s.id) === 'multilingual',
);
expect(hasMultilingual).toBe(false);
});
it('should merge multilingual strategy languages with existing language config', () => {
const config = {
plugins: ['default'],
strategies: [{ id: 'multilingual', config: { languages: ['de', 'it'] } }],
language: 'ja',
};
const result = RedteamConfigSchema.parse(config);
// Should merge existing language with 'en' and multilingual languages
expect(result.language).toEqual(['ja', 'en', 'de', 'it']);
});
});
describe('pluginOptions', () => {
it('should not contain duplicate entries', () => {
const duplicates = pluginOptions.filter((item, index) => pluginOptions.indexOf(item) !== index);
expect(duplicates).toEqual([]);
});
it('should contain unique values from all source arrays', () => {
const manualSet = new Set([...COLLECTIONS, ...REDTEAM_ALL_PLUGINS, ...ALIASED_PLUGINS]);
expect(pluginOptions).toHaveLength(manualSet.size);
expect(new Set(pluginOptions).size).toBe(pluginOptions.length);
});
it('should be sorted alphabetically', () => {
const sortedCopy = [...pluginOptions].sort();
expect(pluginOptions).toEqual(sortedCopy);
});
it('should contain all expected plugin types', () => {
const hasCollectionItem = COLLECTIONS.some((item) => pluginOptions.includes(item));
const hasRedteamPlugin = REDTEAM_ALL_PLUGINS.some((item) => pluginOptions.includes(item));
const hasAliasedPlugin = ALIASED_PLUGINS.some((item) => pluginOptions.includes(item));
expect(hasCollectionItem).toBe(true);
expect(hasRedteamPlugin).toBe(true);
expect(hasAliasedPlugin).toBe(true);
});
it('should handle the specific bias duplicate case', () => {
const biasCount = pluginOptions.filter((option) => option === 'bias').length;
expect(biasCount).toBe(1);
});
it('should deduplicate when source arrays have overlapping values', () => {
// This test is specifically for the bias duplicate case
const biasInCollections = COLLECTIONS.includes('bias' as any);
const biasInAliased = ALIASED_PLUGINS.includes('bias' as any);
expect(biasInCollections).toBe(true);
expect(biasInAliased).toBe(true);
const biasOccurrences = pluginOptions.filter((p) => p === 'bias');
expect(biasOccurrences).toHaveLength(1);
});
});
describe('redteam validators', () => {
describe('strategyIdSchema', () => {
describe('valid built-in strategies', () => {
it('should accept standard built-in strategies', () => {
const validStrategies = [
'basic',
'jailbreak',
'crescendo',
'goat',
'multilingual',
'base64',
'hex',
'rot13',
];
validStrategies.forEach((strategy) => {
expect(() => strategyIdSchema.parse(strategy)).not.toThrow();
});
});
});
describe('custom strategy validation', () => {
it('should accept simple custom strategy', () => {
expect(() => strategyIdSchema.parse('custom')).not.toThrow();
});
it('should accept custom strategy variants with compound IDs', () => {
const customVariants = [
'custom:aggressive',
'custom:greeting-strategy',
'custom:multi-word-variant',
'custom:snake_case_variant',
'custom:variant123',
'custom:CamelCaseVariant',
'custom:variant.with.dots',
'custom:very-long-complex-variant-name-with-many-hyphens',
];
customVariants.forEach((variant) => {
expect(() => strategyIdSchema.parse(variant)).not.toThrow();
});
});
it('should accept edge case custom strategy formats', () => {
// These are actually valid custom strategies according to isCustomStrategy
const validCustomStrategies = [
'custom:', // Valid - empty variant
'custom::', // Valid - double colon variant
'custom:variant:', // Valid - trailing colon variant
];
validCustomStrategies.forEach((strategy) => {
expect(() => strategyIdSchema.parse(strategy)).not.toThrow();
});
// Test invalid strategies that don't match any pattern
const invalidStrategies = [
':custom', // Invalid - doesn't start with 'custom'
'invalid-strategy-123', // Invalid - not a built-in strategy
];
invalidStrategies.forEach((strategy) => {
expect(() => strategyIdSchema.parse(strategy)).toThrow(z.ZodError);
});
});
});
describe('file-based strategy validation', () => {
it('should accept valid file:// strategy paths', () => {
const validFilePaths = [
'file://strategy.js',
'file://strategy.ts',
'file://./strategy.js',
'file:///absolute/path/strategy.ts',
'file://relative/path/strategy.js',
];
validFilePaths.forEach((path) => {
expect(() => strategyIdSchema.parse(path)).not.toThrow();
});
});
it('should reject invalid file:// strategy paths', () => {
const invalidFilePaths = [
'file://strategy.txt',
'file://strategy.py',
'file://strategy',
'strategy.js',
'file:/strategy.js',
];
invalidFilePaths.forEach((path) => {
let error: z.ZodError | undefined;
expect(() => {
try {
strategyIdSchema.parse(path);
} catch (e) {
if (e instanceof z.ZodError) {
error = e;
}
throw e;
}
}).toThrow(z.ZodError);
expect(error).toBeDefined();
const message = error?.issues[0].message || '';
// Zod v4 may return "Invalid input" for union failures
const hasValidMessage =
message.includes('Custom strategies must start with file:// and end with .js or .ts') ||
message.includes('Invalid input');
expect(hasValidMessage).toBe(true);
});
});
});
describe('invalid strategy names', () => {
it('should reject unknown strategy names', () => {
// Test string strategies
const invalidStringStrategies = [
'unknown-strategy',
'invalid_strategy',
'not-a-strategy',
'fakeStrategy',
];
invalidStringStrategies.forEach((strategy) => {
let error: z.ZodError | undefined;
expect(() => {
try {
strategyIdSchema.parse(strategy);
} catch (e) {
if (e instanceof z.ZodError) {
error = e;
}
throw e;
}
}).toThrow(z.ZodError);
expect(error).toBeDefined();
const message = error?.issues[0].message || '';
const validMessages = [
'Custom strategies must start with file:// and end with .js or .ts',
'Strategy must be one of the built-in strategies:',
'Invalid enum value',
'Invalid input', // Zod v4 generic union error
];
const hasValidMessage = validMessages.some((msg) => message.includes(msg));
expect(hasValidMessage).toBe(true);
});
// Test null/undefined/empty separately
expect(() => strategyIdSchema.parse(null)).toThrow(z.ZodError);
expect(() => strategyIdSchema.parse(undefined)).toThrow(z.ZodError);
expect(() => strategyIdSchema.parse('')).toThrow(z.ZodError);
});
});
});
describe('RedteamStrategySchema', () => {
describe('string format strategies', () => {
it('should accept valid strategy strings', () => {
const validStrategies = [
'basic',
'custom',
'custom:variant',
'jailbreak',
'crescendo',
'file://strategy.js',
];
validStrategies.forEach((strategy) => {
expect(() => RedteamStrategySchema.parse(strategy)).not.toThrow();
});
});
});
describe('object format strategies', () => {
it('should accept strategy objects with valid IDs', () => {
const validStrategyObjects = [
{ id: 'basic' },
{ id: 'custom' },
{ id: 'custom:aggressive' },
{ id: 'jailbreak', config: { enabled: true } },
{ id: 'custom:greeting', config: { strategyText: 'Be polite', stateful: true } },
{ id: 'multilingual', config: { languages: ['es', 'fr'] } },
{ id: 'file://custom.js', config: { param: 'value' } },
];
validStrategyObjects.forEach((strategy) => {
expect(() => RedteamStrategySchema.parse(strategy)).not.toThrow();
});
});
it('should accept custom strategy objects with complex configurations', () => {
const customStrategyWithConfig = {
id: 'custom:complex-variant',
config: {
strategyText:
'If current round is 0, generatedQuestion should be just "hi" by itself...',
stateful: true,
temperature: 0.8,
maxTokens: 100,
customParam: true,
nestedConfig: {
subParam: 'value',
subArray: [1, 2, 3],
},
},
};
expect(() => RedteamStrategySchema.parse(customStrategyWithConfig)).not.toThrow();
});
it('should reject strategy objects with invalid IDs', () => {
// Test objects with invalid IDs
const invalidIdObjects = [
{ id: 'custom-invalid' },
{ id: 'notcustom:variant' },
{ id: 'file://invalid.txt' },
{ id: '' },
];
invalidIdObjects.forEach((strategy) => {
let error: z.ZodError | undefined;
expect(() => {
try {
RedteamStrategySchema.parse(strategy);
} catch (e) {
if (e instanceof z.ZodError) {
error = e;
}
throw e;
}
}).toThrow(z.ZodError);
expect(error).toBeDefined();
const message = error?.issues[0].message || '';
const validMessages = [
'Custom strategies must start with file:// and end with .js or .ts',
'Strategy must be one of the built-in strategies:',
'Invalid enum value',
'Invalid input', // Zod v4 generic union error
];
const hasValidMessage = validMessages.some((msg) => message.includes(msg));
expect(hasValidMessage).toBe(true);
});
// Test object without id separately
expect(() => RedteamStrategySchema.parse({})).toThrow(z.ZodError);
});
});
describe('edge cases', () => {
it('should handle strategy objects with empty config', () => {
const strategyWithEmptyConfig = { id: 'custom:variant', config: {} };
expect(() => RedteamStrategySchema.parse(strategyWithEmptyConfig)).not.toThrow();
});
it('should handle strategy objects with undefined config', () => {
const strategyWithUndefinedConfig = { id: 'custom:variant', config: undefined };
expect(() => RedteamStrategySchema.parse(strategyWithUndefinedConfig)).not.toThrow();
});
it('should handle strategy objects without config property', () => {
const strategyWithoutConfig = { id: 'custom:variant' };
expect(() => RedteamStrategySchema.parse(strategyWithoutConfig)).not.toThrow();
});
});
describe('error messages', () => {
it('should provide helpful error message for invalid file strategy', () => {
let error: z.ZodError | undefined;
expect(() => {
try {
strategyIdSchema.parse('file://strategy.txt');
} catch (e) {
if (e instanceof z.ZodError) {
error = e;
}
throw e;
}
}).toThrow(z.ZodError);
expect(error).toBeDefined();
const message = error?.issues[0].message || '';
// Zod v4 may return "Invalid input" for union failures or the specific message
const hasValidMessage =
message.includes('Custom strategies must start with file:// and end with .js or .ts') ||
message.includes('Invalid input');
expect(hasValidMessage).toBe(true);
});
it('should provide helpful error message for completely invalid strategy', () => {
let error: z.ZodError | undefined;
expect(() => {
try {
strategyIdSchema.parse('totally-invalid-strategy');
} catch (e) {
if (e instanceof z.ZodError) {
error = e;
}
throw e;
}
}).toThrow(z.ZodError);
expect(error).toBeDefined();
const message = error?.issues[0].message || '';
const validMessages = [
'Custom strategies must start with file:// and end with .js or .ts',
'Strategy must be one of the built-in strategies:',
'Invalid enum value',
'Invalid input', // Zod v4 generic union error
];
const hasValidMessage = validMessages.some((msg) => message.includes(msg));
expect(hasValidMessage).toBe(true);
});
});
describe('integration with actual use cases', () => {
it('should validate realistic custom strategy configurations', () => {
const realisticConfigurations = [
'custom:greeting-strategy',
'custom:default-strategy',
{
id: 'custom:greeting-strategy',
config: {
stateful: true,
strategyText:
"If current round is 0, generatedQuestion should be just 'Hello can you help me?' by itself...",
},
},
{
id: 'custom:default-strategy',
config: {
stateful: true,
strategyText:
"If current round is 0, generatedQuestion should be just 'hi' by itself...",
},
},
];
realisticConfigurations.forEach((config) => {
expect(() => RedteamStrategySchema.parse(config)).not.toThrow();
});
});
it('should validate mixed strategy arrays like those used in real configurations', () => {
const mixedStrategies = [
'basic',
{ id: 'jailbreak', config: { enabled: true } },
'custom:aggressive-variant',
{
id: 'custom:polite-variant',
config: {
strategyText: 'Be very polite and formal',
stateful: false,
},
},
'crescendo',
'file://./my-custom-strategy.js',
];
mixedStrategies.forEach((strategy) => {
expect(() => RedteamStrategySchema.parse(strategy)).not.toThrow();
});
});
});
});
});
describe('layer strategy deduplication', () => {
it('should keep multiple layer strategies with different labels', () => {
const config = {
plugins: ['default'],
strategies: [
{ id: 'layer', config: { label: 'hydra-audio', steps: ['jailbreak:hydra', 'audio'] } },
{ id: 'layer', config: { label: 'crescendo-audio', steps: ['crescendo', 'audio'] } },
],
};
const result = RedteamConfigSchema.parse(config);
// Both layer strategies should be kept (different labels)
const layerStrategies = result.strategies?.filter(
(s) => typeof s !== 'string' && s.id === 'layer',
);
expect(layerStrategies).toHaveLength(2);
});
it('should keep multiple layer strategies with different steps', () => {
const config = {
plugins: ['default'],
strategies: [
{ id: 'layer', config: { steps: ['jailbreak:hydra', 'audio'] } },
{ id: 'layer', config: { steps: ['crescendo', 'image'] } },
],
};
const result = RedteamConfigSchema.parse(config);
// Both layer strategies should be kept (different steps)
const layerStrategies = result.strategies?.filter(
(s) => typeof s !== 'string' && s.id === 'layer',
);
expect(layerStrategies).toHaveLength(2);
});
it('should deduplicate identical layer strategies', () => {
const config = {
plugins: ['default'],
strategies: [
{ id: 'layer', config: { label: 'same-label', steps: ['jailbreak:hydra', 'audio'] } },
{ id: 'layer', config: { label: 'same-label', steps: ['jailbreak:hydra', 'audio'] } },
],
};
const result = RedteamConfigSchema.parse(config);
// Duplicate layer strategies should be deduplicated
const layerStrategies = result.strategies?.filter(
(s) => typeof s !== 'string' && s.id === 'layer',
);
expect(layerStrategies).toHaveLength(1);
});
it('should use label for deduplication key when provided', () => {
const config = {
plugins: ['default'],
strategies: [
{ id: 'layer', config: { label: 'unique-label', steps: ['jailbreak:hydra', 'audio'] } },
{ id: 'layer', config: { label: 'unique-label', steps: ['crescendo', 'image'] } }, // Same label, different steps
],
};
const result = RedteamConfigSchema.parse(config);
// Should only keep one (label is the key, so second overwrites first)
const layerStrategies = result.strategies?.filter(
(s) => typeof s !== 'string' && s.id === 'layer',
);
expect(layerStrategies).toHaveLength(1);
// The last one wins
expect(layerStrategies?.[0]).toEqual(
expect.objectContaining({
config: expect.objectContaining({ steps: ['crescendo', 'image'] }),
}),
);
});
it('should handle layer strategies without label or steps', () => {
const config = {
plugins: ['default'],
strategies: [{ id: 'layer', config: {} }],
};
const result = RedteamConfigSchema.parse(config);
const layerStrategies = result.strategies?.filter(
(s) => typeof s !== 'string' && s.id === 'layer',
);
expect(layerStrategies).toHaveLength(1);
});
it('should handle mixed layer and non-layer strategies', () => {
const config = {
plugins: ['default'],
strategies: [
'basic',
{ id: 'layer', config: { label: 'audio-attack', steps: ['jailbreak:hydra', 'audio'] } },
'jailbreak',
{ id: 'layer', config: { label: 'image-attack', steps: ['crescendo', 'image'] } },
],
};
const result = RedteamConfigSchema.parse(config);
// Should have all strategies
expect(result.strategies?.length).toBeGreaterThanOrEqual(3);
// Layer strategies should both be present
const layerStrategies = result.strategies?.filter(
(s) => typeof s !== 'string' && s.id === 'layer',
);
expect(layerStrategies).toHaveLength(2);
});
});
describe('Error message quality', () => {
it('should provide clear error messages for common mistakes', () => {
// Test various common mistakes and ensure error messages are helpful
// Note: Zod v4 may return "Invalid input" for union failures
const testCases = [
{
config: { plugins: ['non-existent-plugin'] },
expectedError: /Custom plugins must start with file:\/\/|Invalid input/,
description: 'non-existent plugin name',
},
{
config: { strategies: ['file://strategy.json'] },
expectedError:
/Custom strategies must start with file:\/\/ and end with .js or .ts|Invalid input/,
description: 'wrong file extension for strategy',
},
{
config: { strategies: ['invalid-strategy-name'] },
expectedError:
/Custom strategies must start with file:\/\/ and end with .js or .ts|Invalid input/,
description: 'invalid strategy name',
},
{
config: { plugins: [{ id: 'default' }], numTests: 'many' as any },
expectedError: /Expected number|Invalid type|Invalid input|expected number/i,
description: 'string instead of number for numTests',
},
{
config: { plugins: ['default'], delay: 'slow' as any },
expectedError: /Expected number|Invalid type|Invalid input|expected number/i,
description: 'string instead of number for delay',
},
];
testCases.forEach(({ config, expectedError }) => {
let errorCaught = false;
let zodError: z.ZodError | null = null;
try {
RedteamConfigSchema.parse(config);
} catch (error) {
errorCaught = true;
if (error instanceof z.ZodError) {
zodError = error;
}
}
expect(errorCaught).toBe(true);
expect(zodError).not.toBeNull();
const message = zodError!.issues[0]?.message || '';
// Always perform the assertion, the expected pattern handles both string and regex
const matches =
typeof expectedError === 'string'
? message.includes(expectedError)
: expectedError.test(message);
expect(matches).toBe(true);
});
});
});
+90
View File
@@ -0,0 +1,90 @@
import { describe, expect, it } from 'vitest';
import { NunjucksFilterMapSchema } from '../../src/validators/shared';
describe('NunjucksFilterMapSchema', () => {
it('should accept empty object', () => {
const result = NunjucksFilterMapSchema.safeParse({});
expect(result.success).toBe(true);
expect(result.data).toEqual({});
});
it('should accept valid filter map with functions', () => {
const input = {
uppercase: (val: unknown) => String(val).toUpperCase(),
lowercase: (val: unknown) => String(val).toLowerCase(),
};
const result = NunjucksFilterMapSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toHaveProperty('uppercase');
expect(result.data).toHaveProperty('lowercase');
});
it('should accept single filter', () => {
const input = {
trim: (val: unknown) => String(val).trim(),
};
const result = NunjucksFilterMapSchema.safeParse(input);
expect(result.success).toBe(true);
expect(result.data).toHaveProperty('trim');
});
it('should reject non-function values', () => {
const result = NunjucksFilterMapSchema.safeParse({
notAFunction: 'string-value',
});
expect(result.success).toBe(false);
});
it('should reject number values', () => {
const result = NunjucksFilterMapSchema.safeParse({
badFilter: 42,
});
expect(result.success).toBe(false);
});
it('should reject null values', () => {
const result = NunjucksFilterMapSchema.safeParse({
nullFilter: null,
});
expect(result.success).toBe(false);
});
it('should reject boolean values', () => {
const result = NunjucksFilterMapSchema.safeParse({
boolFilter: true,
});
expect(result.success).toBe(false);
});
it('should accept functions with multiple arguments', () => {
const input = {
replace: (val: unknown, search: unknown, replacement: unknown) =>
String(val).replace(String(search), String(replacement)),
};
const result = NunjucksFilterMapSchema.safeParse(input);
expect(result.success).toBe(true);
});
it('should reject non-object input', () => {
expect(NunjucksFilterMapSchema.safeParse('not-an-object').success).toBe(false);
expect(NunjucksFilterMapSchema.safeParse(42).success).toBe(false);
expect(NunjucksFilterMapSchema.safeParse(null).success).toBe(false);
expect(NunjucksFilterMapSchema.safeParse(undefined).success).toBe(false);
});
it('should reject array input', () => {
const result = NunjucksFilterMapSchema.safeParse([() => 'test']);
expect(result.success).toBe(false);
});
it('should accept mix of valid function filters', () => {
const input = {
first: () => 'a',
second: (_a: unknown, _b: unknown) => 'b',
third: (..._args: unknown[]) => 'c',
};
const result = NunjucksFilterMapSchema.safeParse(input);
expect(result.success).toBe(true);
expect(Object.keys(result.data!)).toHaveLength(3);
});
});
+259
View File
@@ -0,0 +1,259 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
vi.mock('../../src/logger', () => ({
default: {
debug: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
},
}));
import {
determineEffectiveSessionSource,
formatConfigBody,
formatConfigHeaders,
validateSessionConfig,
} from '../../src/validators/util';
import type { ApiProvider } from '../../src/types/providers';
afterEach(() => {
vi.restoreAllMocks();
});
describe('formatConfigBody', () => {
it('should return "None configured" when body is falsy', () => {
expect(formatConfigBody({ body: undefined })).toBe('None configured');
expect(formatConfigBody({ body: null })).toBe('None configured');
expect(formatConfigBody({ body: '' })).toBe('None configured');
});
it('should pretty-print valid JSON string body', () => {
const body = '{"key":"value","nested":{"a":1}}';
const result = formatConfigBody({ body });
expect(result).toContain('"key": "value"');
expect(result).toContain('"nested"');
// Should be indented
expect(result).toContain(' ');
});
it('should indent non-JSON string body', () => {
const body = 'plain text body';
const result = formatConfigBody({ body });
expect(result).toBe('\n plain text body');
});
it('should stringify and indent object body', () => {
const body = { key: 'value', number: 42 };
const result = formatConfigBody({ body });
expect(result).toContain('"key": "value"');
expect(result).toContain('"number": 42');
expect(result).toContain(' ');
});
it('should handle nested objects', () => {
const body = { outer: { inner: 'deep' } };
const result = formatConfigBody({ body });
expect(result).toContain('"outer"');
expect(result).toContain('"inner": "deep"');
});
it('should handle array body', () => {
const body = [1, 2, 3];
const result = formatConfigBody({ body });
expect(result).toContain('1');
expect(result).toContain('2');
expect(result).toContain('3');
});
});
describe('formatConfigHeaders', () => {
it('should return "None configured" when headers is undefined', () => {
expect(formatConfigHeaders({ headers: undefined })).toBe('None configured');
});
it('should format headers as key-value pairs', () => {
const headers = {
'Content-Type': 'application/json',
Authorization: 'Bearer token123',
};
const result = formatConfigHeaders({ headers });
expect(result).toContain('Content-Type: application/json');
expect(result).toContain('Authorization: Bearer token123');
});
it('should handle single header', () => {
const result = formatConfigHeaders({ headers: { Accept: 'text/plain' } });
expect(result).toContain('Accept: text/plain');
});
it('should handle empty headers object', () => {
const result = formatConfigHeaders({ headers: {} });
// Empty object should produce a newline with no entries
expect(result).toBe('\n');
});
});
describe('validateSessionConfig', () => {
it('should warn when sessionId template is missing from config', () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
// The function uses the logger module, so we just verify it doesn't throw
const provider = {
id: () => 'test-provider',
config: { url: 'https://example.com/api' },
callApi: vi.fn(),
} as unknown as ApiProvider;
// The function uses the logger module, so we just verify it doesn't throw
validateSessionConfig({
provider,
sessionSource: 'client',
});
warnSpy.mockRestore();
});
it('should not warn when sessionId template is present in config', () => {
const provider = {
id: () => 'test-provider',
config: {
url: 'https://example.com/api',
headers: { 'X-Session': '{{sessionId}}' },
},
callApi: vi.fn(),
} as unknown as ApiProvider;
// Should not throw
expect(() =>
validateSessionConfig({
provider,
sessionSource: 'client',
}),
).not.toThrow();
});
it('should handle server session source without session parser', () => {
const provider = {
id: () => 'test-provider',
config: { url: 'https://example.com/api', body: '{{sessionId}}' },
callApi: vi.fn(),
} as unknown as ApiProvider;
// Should not throw even with missing session parser
expect(() =>
validateSessionConfig({
provider,
sessionSource: 'server',
}),
).not.toThrow();
});
it('should handle server session source with session parser', () => {
const provider = {
id: () => 'test-provider',
config: { url: 'https://example.com/api', body: '{{sessionId}}' },
callApi: vi.fn(),
} as unknown as ApiProvider;
expect(() =>
validateSessionConfig({
provider,
sessionSource: 'server',
sessionConfig: { sessionParser: 'response.headers.x-session-id' },
}),
).not.toThrow();
});
it('should handle provider with no config', () => {
const provider = {
id: () => 'test-provider',
config: {},
callApi: vi.fn(),
} as unknown as ApiProvider;
expect(() =>
validateSessionConfig({
provider,
sessionSource: 'client',
}),
).not.toThrow();
});
});
describe('determineEffectiveSessionSource', () => {
it('should use sessionConfig.sessionSource when provided', () => {
const provider = {
id: () => 'test-provider',
config: { sessionSource: 'client' },
callApi: vi.fn(),
} as unknown as ApiProvider;
const result = determineEffectiveSessionSource({
provider,
sessionConfig: { sessionSource: 'server' },
});
expect(result).toBe('server');
});
it('should use provider.config.sessionSource when sessionConfig is not provided', () => {
const provider = {
id: () => 'test-provider',
config: { sessionSource: 'server' },
callApi: vi.fn(),
} as unknown as ApiProvider;
const result = determineEffectiveSessionSource({ provider });
expect(result).toBe('server');
});
it('should return "server" when provider has sessionParser', () => {
const provider = {
id: () => 'test-provider',
config: { sessionParser: 'response.id' },
callApi: vi.fn(),
} as unknown as ApiProvider;
const result = determineEffectiveSessionSource({ provider });
expect(result).toBe('server');
});
it('should default to "client" when no session config exists', () => {
const provider = {
id: () => 'test-provider',
config: {},
callApi: vi.fn(),
} as unknown as ApiProvider;
const result = determineEffectiveSessionSource({ provider });
expect(result).toBe('client');
});
it('should prioritize sessionConfig over provider config', () => {
const provider = {
id: () => 'test-provider',
config: { sessionSource: 'client', sessionParser: 'response.id' },
callApi: vi.fn(),
} as unknown as ApiProvider;
const result = determineEffectiveSessionSource({
provider,
sessionConfig: { sessionSource: 'server' },
});
expect(result).toBe('server');
});
it('should handle undefined sessionConfig', () => {
const provider = {
id: () => 'test-provider',
config: {},
callApi: vi.fn(),
} as unknown as ApiProvider;
const result = determineEffectiveSessionSource({
provider,
sessionConfig: undefined,
});
expect(result).toBe('client');
});
});