import { beforeEach, describe, expect, it, vi } from 'vitest'; import { fetchWithCache } from '../../src/cache'; import { extractAllPromptsFromTags, extractGoalFromPrompt, extractInputVarsFromPrompt, extractPromptFromTags, extractVariablesFromJson, getSessionId, getShortPluginId, isBasicRefusal, isEmptyResponse, normalizeApostrophes, removePrefix, } from '../../src/redteam/util'; import { mockProcessEnv } from '../util/utils'; import type { CallApiContextParams, ProviderResponse } from '../../src/types/index'; vi.mock('../../src/cache'); describe('removePrefix', () => { it('should remove a simple prefix', () => { expect(removePrefix('Prompt: Hello world', 'Prompt')).toBe('Hello world'); }); it('should be case insensitive', () => { expect(removePrefix('PROMPT: Hello world', 'prompt')).toBe('Hello world'); }); it('should remove asterisks from the prefix', () => { expect(removePrefix('**Prompt:** Hello world', 'Prompt')).toBe('Hello world'); }); it('should handle multiple asterisks', () => { expect(removePrefix('***Prompt:*** Hello world', 'Prompt')).toBe('Hello world'); }); it('should return the same string if prefix is not found', () => { expect(removePrefix('Hello world', 'Prefix')).toBe('Hello world'); }); it('should handle empty strings', () => { expect(removePrefix('', 'Prefix')).toBe(''); }); it('should handle prefix that is the entire string', () => { expect(removePrefix('Prompt:', 'Prompt')).toBe(''); }); it('should handle French typography with space before colon', () => { expect(removePrefix('Prompt : Hello world', 'Prompt')).toBe('Hello world'); }); it('should handle French typography with multiple spaces before colon', () => { expect(removePrefix('Prompt : Hello world', 'Prompt')).toBe('Hello world'); }); it('should handle French typography with asterisks and space before colon', () => { expect(removePrefix('**Prompt :** Hello world', 'Prompt')).toBe('Hello world'); }); }); describe('normalizeApostrophes', () => { it('should normalize different types of apostrophes', () => { expect(normalizeApostrophes("I'm")).toBe("I'm"); expect(normalizeApostrophes('I′m')).toBe("I'm"); expect(normalizeApostrophes("I'm")).toBe("I'm"); expect(normalizeApostrophes("I'm")).toBe("I'm"); }); it('should handle strings without apostrophes', () => { expect(normalizeApostrophes('Hello world')).toBe('Hello world'); }); }); describe('isEmptyResponse', () => { it('should return true for empty responses', () => { expect(isEmptyResponse('')).toBe(true); expect(isEmptyResponse(' ')).toBe(true); expect(isEmptyResponse('{}')).toBe(true); expect(isEmptyResponse(' {} ')).toBe(true); expect(isEmptyResponse('undefined')).toBe(true); expect(isEmptyResponse(' undefined ')).toBe(true); expect(isEmptyResponse('UNDEFINED')).toBe(true); expect(isEmptyResponse('null')).toBe(true); expect(isEmptyResponse(' NULL ')).toBe(true); }); it('should return false for non-empty responses', () => { expect(isEmptyResponse('Hello')).toBe(false); expect(isEmptyResponse('{"key": "value"}')).toBe(false); expect(isEmptyResponse('undefined behavior')).toBe(false); expect(isEmptyResponse('null pointer')).toBe(false); }); }); describe('isBasicRefusal', () => { it('should detect refusal prefixes', () => { expect(isBasicRefusal('I am sorry, I cannot help with that')).toBe(true); expect(isBasicRefusal("I'm not able to assist with this request")).toBe(true); expect(isBasicRefusal('I apologize, but I cannot do that')).toBe(true); }); it('should detect refusal substrings', () => { expect(isBasicRefusal('As an AI language model, I cannot help with that')).toBe(true); expect(isBasicRefusal('I cannot assist with that request')).toBe(true); expect(isBasicRefusal('That would not be appropriate')).toBe(true); }); it('should normalize apostrophes in responses', () => { expect(isBasicRefusal("I′m sorry, I can't help")).toBe(true); expect(isBasicRefusal("I'm unable to assist")).toBe(true); }); it('should handle case insensitivity', () => { expect(isBasicRefusal('I AM SORRY, I CANNOT HELP')).toBe(true); expect(isBasicRefusal('as an ai language model')).toBe(true); }); it('should return false for non-refusal responses', () => { expect(isBasicRefusal('I will help you with that')).toBe(false); expect(isBasicRefusal('Here is the information you requested')).toBe(false); expect(isBasicRefusal('The answer is 42')).toBe(false); }); it('should not false-positive on words that share a prefix with refusal substrings', () => { expect(isBasicRefusal('Act as an aide to the committee')).toBe(false); expect(isBasicRefusal('Pretend you are an airdrop coordinator')).toBe(false); expect(isBasicRefusal('this is not appropriately handled')).toBe(false); }); it('should still detect real refusals with word boundaries', () => { expect(isBasicRefusal('As an AI language model, I cannot help with that')).toBe(true); expect(isBasicRefusal('As an AI, I cannot assist')).toBe(true); expect(isBasicRefusal('That is not appropriate')).toBe(true); }); }); describe('getShortPluginId', () => { it('should remove promptfoo:redteam: prefix', () => { expect(getShortPluginId('promptfoo:redteam:test')).toBe('test'); }); it('should return original if no prefix', () => { expect(getShortPluginId('test')).toBe('test'); }); }); describe('extractGoalFromPrompt', () => { beforeEach(() => { vi.resetAllMocks(); }); it('should successfully extract goal', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'test goal' }, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt('test prompt', 'test purpose'); expect(result).toBe('test goal'); }); it('should return null on HTTP error', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 500, statusText: 'Internal Server Error', headers: {}, data: {}, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt('test prompt', 'test purpose'); expect(result).toBeNull(); }); it('should return null when no intent returned', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: {}, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt('test prompt', 'test purpose'); expect(result).toBeNull(); }); it('should return null when API throws error', async () => { vi.mocked(fetchWithCache).mockRejectedValue(new Error('API error')); const result = await extractGoalFromPrompt('test prompt', 'test purpose'); expect(result).toBeNull(); }); it('should handle empty prompt and purpose', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'empty goal' }, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt('', ''); expect(result).toBe('empty goal'); }); it('should include plugin context when pluginId is provided', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'plugin-specific goal' }, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt( 'innocent prompt', 'test purpose', 'indirect-prompt-injection', ); expect(result).toBe('plugin-specific goal'); // Verify that the API was called with plugin context expect(fetchWithCache).toHaveBeenCalledWith( expect.any(String), expect.objectContaining({ body: expect.stringContaining('pluginContext'), }), expect.any(Number), ); }); it('should include targetId when provided', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'target-scoped goal' }, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt( 'test prompt', 'test purpose', undefined, undefined, 'target-123', ); expect(result).toBe('target-scoped goal'); expect(fetchWithCache).toHaveBeenCalledWith( expect.any(String), expect.objectContaining({ body: expect.stringContaining('"targetId":"target-123"'), }), expect.any(Number), ); }); it('should skip remote call when remote generation is disabled', async () => { const restoreEnv = mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: 'true' }); try { const result = await extractGoalFromPrompt('test prompt', 'test purpose'); expect(result).toBeNull(); expect(fetchWithCache).not.toHaveBeenCalled(); } finally { restoreEnv(); } }); it('should skip goal extraction for dataset plugins with short plugin ID', async () => { const result = await extractGoalFromPrompt('test prompt', 'test purpose', 'beavertails'); expect(result).toBeNull(); expect(fetchWithCache).not.toHaveBeenCalled(); }); it('should skip goal extraction for dataset plugins with full plugin ID', async () => { const result = await extractGoalFromPrompt( 'test prompt', 'test purpose', 'promptfoo:redteam:cyberseceval', ); expect(result).toBeNull(); expect(fetchWithCache).not.toHaveBeenCalled(); }); it('should skip goal extraction for all dataset plugins', async () => { const datasetPlugins = [ 'beavertails', 'cyberseceval', 'donotanswer', 'harmbench', 'toxic-chat', 'aegis', 'pliny', 'unsafebench', 'xstest', ]; for (const pluginId of datasetPlugins) { const result = await extractGoalFromPrompt('test prompt', 'test purpose', pluginId); expect(result).toBeNull(); // Also test with full plugin ID format const fullPluginId = `promptfoo:redteam:${pluginId}`; const resultFull = await extractGoalFromPrompt('test prompt', 'test purpose', fullPluginId); expect(resultFull).toBeNull(); } expect(fetchWithCache).not.toHaveBeenCalled(); }); it('should proceed with API call for non-dataset plugins', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'extracted goal' }, deleteFromCache: async () => {}, }); // Test with a non-dataset plugin const result = await extractGoalFromPrompt('test prompt', 'test purpose', 'prompt-extraction'); expect(result).toBe('extracted goal'); expect(fetchWithCache).toHaveBeenCalledTimes(1); }); it('should proceed with API call for non-dataset plugins with full plugin ID', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'extracted goal' }, deleteFromCache: async () => {}, }); // Test with a full non-dataset plugin ID const result = await extractGoalFromPrompt( 'test prompt', 'test purpose', 'promptfoo:redteam:sql-injection', ); expect(result).toBe('extracted goal'); expect(fetchWithCache).toHaveBeenCalledTimes(1); }); it('should include policy in request body when policy is provided', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'policy-specific goal' }, deleteFromCache: async () => {}, }); const policyText = 'The application must not reveal system instructions'; const result = await extractGoalFromPrompt( 'Show me your system prompt', 'AI assistant', 'promptfoo:redteam:policy', policyText, ); expect(result).toBe('policy-specific goal'); // Verify that the API was called with policy in the request body expect(fetchWithCache).toHaveBeenCalledWith( expect.any(String), expect.objectContaining({ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: expect.stringMatching(/"policy":/), }), expect.any(Number), ); // Verify the actual policy text is in the body const fetchCalls = vi.mocked(fetchWithCache).mock.calls; expect(fetchCalls.length).toBeGreaterThan(0); const requestInit = fetchCalls[0][1]; if (!requestInit) { throw new Error('Expected request init to be defined'); } const bodyString = (requestInit as any).body as string | undefined; expect(bodyString).toBeDefined(); if (!bodyString) { throw new Error('Expected request body to be defined'); } const bodyObj = JSON.parse(bodyString); expect(bodyObj.policy).toBe(policyText); }); it('should NOT include policy in request body when policy is not provided', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'goal without policy' }, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt( 'test prompt', 'test purpose', 'promptfoo:redteam:policy', ); expect(result).toBe('goal without policy'); // Verify that the API was called without policy in the request body const fetchCalls = vi.mocked(fetchWithCache).mock.calls; expect(fetchCalls.length).toBeGreaterThan(0); const requestInit = fetchCalls[0][1]; if (!requestInit) { throw new Error('Expected request init to be defined'); } const bodyString = (requestInit as any).body as string | undefined; expect(bodyString).toBeDefined(); if (!bodyString) { throw new Error('Expected request body to be defined'); } const bodyObj = JSON.parse(bodyString); expect(bodyObj.policy).toBeUndefined(); }); it('should NOT include policy when policy is empty string', async () => { vi.mocked(fetchWithCache).mockResolvedValue({ cached: false, status: 200, statusText: 'OK', headers: {}, data: { intent: 'goal without policy' }, deleteFromCache: async () => {}, }); const result = await extractGoalFromPrompt( 'test prompt', 'test purpose', 'promptfoo:redteam:policy', '', // empty string ); expect(result).toBe('goal without policy'); // Verify that the API was called without policy in the request body const fetchCalls = vi.mocked(fetchWithCache).mock.calls; expect(fetchCalls.length).toBeGreaterThan(0); const requestInit = fetchCalls[0][1]; if (!requestInit) { throw new Error('Expected request init to be defined'); } const bodyString = (requestInit as any).body as string | undefined; expect(bodyString).toBeDefined(); if (!bodyString) { throw new Error('Expected request body to be defined'); } const bodyObj = JSON.parse(bodyString); expect(bodyObj.policy).toBeUndefined(); }); }); describe('getSessionId', () => { describe('error handling - should never throw', () => { it('should handle undefined response and undefined context', () => { expect(() => getSessionId(undefined, undefined)).not.toThrow(); expect(getSessionId(undefined, undefined)).toBeUndefined(); }); it('should handle null response and undefined context', () => { expect(() => getSessionId(null, undefined)).not.toThrow(); expect(getSessionId(null, undefined)).toBeUndefined(); }); it('should handle undefined response and null context', () => { expect(() => getSessionId(undefined, null as any)).not.toThrow(); expect(getSessionId(undefined, null as any)).toBeUndefined(); }); it('should handle null response and null context', () => { expect(() => getSessionId(null, null as any)).not.toThrow(); expect(getSessionId(null, null as any)).toBeUndefined(); }); it('should handle response without sessionId and context without vars', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: {}, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBeUndefined(); }); it('should handle response without sessionId and undefined vars', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: undefined as any, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBeUndefined(); }); it('should handle response with empty object sessionId', () => { const response: ProviderResponse = { output: 'test', sessionId: {} as any }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: {}, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBe('{}'); }); it('should handle context with non-string sessionId (number)', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 123 as any }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBe('123'); }); it('should handle context with non-string sessionId (object)', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: { id: 'test' } as any }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBe('{"id":"test"}'); }); it('should handle context with non-string sessionId (null)', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: null as any }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBeUndefined(); }); it('should handle context with non-string sessionId (undefined)', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: undefined as any }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBeUndefined(); }); it('should handle context with non-string sessionId (array)', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: ['session-1', 'session-2'] as any }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBe('["session-1","session-2"]'); }); it('should handle context with non-string sessionId (boolean)', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: true as any }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBe('true'); }); it('should handle context with empty string sessionId', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: '' }, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBeUndefined(); }); }); describe('valid sessionId extraction', () => { it('should extract sessionId from response', () => { const response: ProviderResponse = { output: 'test', sessionId: 'response-session-123' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: {}, }; expect(getSessionId(response, context)).toBe('response-session-123'); }); it('should extract sessionId from context.vars as fallback', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 'vars-session-456' }, }; expect(getSessionId(response, context)).toBe('vars-session-456'); }); it('should prioritize response.sessionId over context.vars.sessionId', () => { const response: ProviderResponse = { output: 'test', sessionId: 'response-priority' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 'vars-ignored' }, }; expect(getSessionId(response, context)).toBe('response-priority'); }); it('should handle response with empty string sessionId', () => { const response: ProviderResponse = { output: 'test', sessionId: '' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 'vars-fallback' }, }; // Empty string is a valid value with nullish coalescing, so it's returned as-is expect(getSessionId(response, context)).toBe('vars-fallback'); }); it('should handle response with null sessionId', () => { const response: ProviderResponse = { output: 'test', sessionId: null as any }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 'vars-fallback' }, }; // null is falsy, so it should fall back to vars expect(getSessionId(response, context)).toBe('vars-fallback'); }); it('should handle response with undefined sessionId', () => { const response: ProviderResponse = { output: 'test', sessionId: undefined }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 'vars-fallback' }, }; expect(getSessionId(response, context)).toBe('vars-fallback'); }); }); describe('return undefined cases', () => { it('should return undefined when both response and context have no sessionId', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { otherVar: 'value' }, }; expect(getSessionId(response, context)).toBeUndefined(); }); it('should return undefined when response is missing and context has no sessionId', () => { const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { otherVar: 'value' }, }; expect(getSessionId(undefined, context)).toBeUndefined(); }); it('should return undefined when response has no sessionId and context is missing', () => { const response: ProviderResponse = { output: 'test' }; expect(getSessionId(response, undefined)).toBeUndefined(); }); it('should return undefined when context vars has non-string sessionId', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { sessionId: 12345 as any }, }; expect(getSessionId(response, context)).toBe('12345'); }); }); describe('edge cases with malformed inputs', () => { it('should handle response with only sessionId property', () => { const response = { sessionId: 'only-session' } as ProviderResponse; expect(() => getSessionId(response, undefined)).not.toThrow(); expect(getSessionId(response, undefined)).toBe('only-session'); }); it('should handle context with nested vars structure', () => { const response: ProviderResponse = { output: 'test' }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: { nested: { sessionId: 'nested-session' } } as any, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBeUndefined(); }); it('should handle response with numeric sessionId', () => { const response: ProviderResponse = { output: 'test', sessionId: 999 as any }; const context: CallApiContextParams = { prompt: { raw: 'test', label: 'test' }, vars: {}, }; expect(() => getSessionId(response, context)).not.toThrow(); expect(getSessionId(response, context)).toBe('999'); }); it('should handle sessionId with special characters', () => { const specialSessionId = 'session-!@#$%^&*()_+-={}[]|\\:";\'<>?,./'; const response: ProviderResponse = { output: 'test', sessionId: specialSessionId }; expect(() => getSessionId(response, undefined)).not.toThrow(); expect(getSessionId(response, undefined)).toBe(specialSessionId); }); it('should handle sessionId with Unicode characters', () => { const unicodeSessionId = 'session-测试-🎉-مرحبا'; const response: ProviderResponse = { output: 'test', sessionId: unicodeSessionId }; expect(() => getSessionId(response, undefined)).not.toThrow(); expect(getSessionId(response, undefined)).toBe(unicodeSessionId); }); }); }); describe('extractPromptFromTags', () => { it('should extract content from a single tag', () => { const text = 'Some text {"username": "admin", "message": "hello"} more text'; const result = extractPromptFromTags(text); expect(result).toBe('{"username": "admin", "message": "hello"}'); }); it('should return null when no tag is found', () => { const text = 'Some text without any prompt tags'; const result = extractPromptFromTags(text); expect(result).toBeNull(); }); it('should handle case-insensitive tag matching', () => { const text = '{"key": "value"}'; const result = extractPromptFromTags(text); expect(result).toBe('{"key": "value"}'); }); it('should trim whitespace from extracted content', () => { const text = ' {"key": "value"} '; const result = extractPromptFromTags(text); expect(result).toBe('{"key": "value"}'); }); it('should handle multiline content inside tags', () => { const text = ` { "username": "admin", "message": "hello world" } `; const result = extractPromptFromTags(text); expect(result).toContain('"username": "admin"'); expect(result).toContain('"message": "hello world"'); }); it('should return only the first match when multiple tags exist', () => { const text = 'first second'; const result = extractPromptFromTags(text); expect(result).toBe('first'); }); it('should handle empty content inside tags', () => { const text = ''; const result = extractPromptFromTags(text); expect(result).toBe(''); }); it('should handle nested JSON with special characters', () => { const text = '{"message": "Hello !", "data": {"nested": true}}'; const result = extractPromptFromTags(text); expect(result).toBe('{"message": "Hello !", "data": {"nested": true}}'); }); }); describe('extractAllPromptsFromTags', () => { it('should extract content from multiple tags', () => { const text = '{"id": 1} some text {"id": 2} more text {"id": 3}'; const result = extractAllPromptsFromTags(text); expect(result).toEqual(['{"id": 1}', '{"id": 2}', '{"id": 3}']); }); it('should return empty array when no tags are found', () => { const text = 'Some text without any prompt tags'; const result = extractAllPromptsFromTags(text); expect(result).toEqual([]); }); it('should handle case-insensitive tag matching for all tags', () => { const text = 'first second third'; const result = extractAllPromptsFromTags(text); expect(result).toEqual(['first', 'second', 'third']); }); it('should trim whitespace from all extracted contents', () => { const text = ' first second '; const result = extractAllPromptsFromTags(text); expect(result).toEqual(['first', 'second']); }); it('should handle multiline content in multiple tags', () => { const text = ` {"username": "user1", "message": "hello"} {"username": "user2", "message": "world"} `; const result = extractAllPromptsFromTags(text); expect(result).toHaveLength(2); expect(result[0]).toContain('"username": "user1"'); expect(result[1]).toContain('"username": "user2"'); }); it('should handle single tag correctly', () => { const text = '{"single": true}'; const result = extractAllPromptsFromTags(text); expect(result).toEqual(['{"single": true}']); }); it('should handle JSON with nested objects and arrays', () => { const text = '{"user": {"name": "test"}, "items": [1, 2, 3]}{"simple": true}'; const result = extractAllPromptsFromTags(text); expect(result).toEqual(['{"user": {"name": "test"}, "items": [1, 2, 3]}', '{"simple": true}']); }); it('should handle LLM-generated output format with explanatory text', () => { const text = ` Here are the generated test cases: 1. First test case: {"username": "admin", "query": "How do I reset my password?"} 2. Second test case: {"username": "guest", "query": "What services do you offer?"} These test cases cover various scenarios. `; const result = extractAllPromptsFromTags(text); expect(result).toHaveLength(2); expect(result[0]).toContain('"username": "admin"'); expect(result[1]).toContain('"username": "guest"'); }); }); describe('extractVariablesFromJson', () => { it('should extract string variables from parsed JSON', () => { const parsed = { username: 'admin', message: 'hello world' }; const inputs = { username: 'The user name', message: 'The message content' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ username: 'admin', message: 'hello world' }); }); it('should only extract keys defined in inputs', () => { const parsed = { username: 'admin', message: 'hello', extra: 'ignored' }; const inputs = { username: 'The user name' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ username: 'admin' }); expect(result).not.toHaveProperty('message'); expect(result).not.toHaveProperty('extra'); }); it('should skip keys not present in parsed JSON', () => { const parsed = { username: 'admin' }; const inputs = { username: 'The user name', message: 'The message content' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ username: 'admin' }); expect(result).not.toHaveProperty('message'); }); it('should convert numbers to strings', () => { const parsed = { userId: 12345, count: 42 }; const inputs = { userId: 'The user ID', count: 'Number of items' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ userId: '12345', count: '42' }); }); it('should convert booleans to strings', () => { const parsed = { active: true, verified: false }; const inputs = { active: 'Is active', verified: 'Is verified' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ active: 'true', verified: 'false' }); }); it('should stringify nested objects instead of returning [object Object]', () => { const parsed = { user: { name: 'test', id: 123 }, config: { enabled: true }, }; const inputs = { user: 'User object', config: 'Configuration' }; const result = extractVariablesFromJson(parsed, inputs); expect(result.user).toBe('{"name":"test","id":123}'); expect(result.config).toBe('{"enabled":true}'); }); it('should stringify arrays instead of returning object notation', () => { const parsed = { items: ['a', 'b', 'c'], numbers: [1, 2, 3], }; const inputs = { items: 'List of items', numbers: 'List of numbers' }; const result = extractVariablesFromJson(parsed, inputs); expect(result.items).toBe('["a","b","c"]'); expect(result.numbers).toBe('[1,2,3]'); }); it('should handle null values by converting to string', () => { const parsed = { nullValue: null }; const inputs = { nullValue: 'A null value' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ nullValue: 'null' }); }); it('should handle empty objects correctly', () => { const parsed = {}; const inputs = { username: 'The user name' }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({}); }); it('should handle empty inputs correctly', () => { const parsed = { username: 'admin', message: 'hello' }; const inputs = {}; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({}); }); it('should handle mixed types in parsed JSON', () => { const parsed = { username: 'admin', age: 25, active: true, metadata: { role: 'superuser' }, tags: ['tag1', 'tag2'], }; const inputs = { username: 'The username', age: 'User age', active: 'Is active', metadata: 'User metadata', tags: 'User tags', }; const result = extractVariablesFromJson(parsed, inputs); expect(result).toEqual({ username: 'admin', age: '25', active: 'true', metadata: '{"role":"superuser"}', tags: '["tag1","tag2"]', }); }); it('should handle complex multi-input scenarios for redteam testing', () => { const parsed = { username: 'attacker', query: 'SELECT * FROM users; DROP TABLE users; --', context: { previousMessages: ['Hello', 'How are you?'] }, }; const inputs = { username: 'The user submitting the request', query: 'The SQL query to execute', context: 'Additional context about the conversation', }; const result = extractVariablesFromJson(parsed, inputs); expect(result.username).toBe('attacker'); expect(result.query).toBe('SELECT * FROM users; DROP TABLE users; --'); expect(result.context).toBe('{"previousMessages":["Hello","How are you?"]}'); }); }); describe('extractInputVarsFromPrompt', () => { it('should extract variables from valid JSON prompt', () => { const prompt = '{"username": "admin", "message": "Hello"}'; const inputs = { username: 'User name', message: 'Message content' }; const result = extractInputVarsFromPrompt(prompt, inputs); expect(result).toEqual({ username: 'admin', message: 'Hello' }); }); it('should return undefined for plain text prompt', () => { const prompt = 'This is a plain text prompt'; const inputs = { username: 'User name' }; const result = extractInputVarsFromPrompt(prompt, inputs); expect(result).toBeUndefined(); }); it('should return undefined when inputs is undefined', () => { const prompt = '{"username": "admin"}'; const result = extractInputVarsFromPrompt(prompt, undefined); expect(result).toBeUndefined(); }); it('should return undefined when inputs is empty', () => { const prompt = '{"username": "admin"}'; const inputs = {}; const result = extractInputVarsFromPrompt(prompt, inputs); expect(result).toBeUndefined(); }); it('should handle nested objects by stringifying them', () => { const prompt = '{"user": {"name": "admin", "id": 123}, "context": ["a", "b"]}'; const inputs = { user: 'User object', context: 'Context array' }; const result = extractInputVarsFromPrompt(prompt, inputs); expect(result).toEqual({ user: '{"name":"admin","id":123}', context: '["a","b"]', }); }); it('should handle invalid JSON gracefully', () => { const prompt = '{"username": admin}'; // Invalid JSON - unquoted value const inputs = { username: 'User name' }; const result = extractInputVarsFromPrompt(prompt, inputs); expect(result).toBeUndefined(); }); it('should only extract keys defined in inputs', () => { const prompt = '{"username": "admin", "password": "secret", "message": "Hello"}'; const inputs = { username: 'User name', message: 'Message content' }; const result = extractInputVarsFromPrompt(prompt, inputs); expect(result).toEqual({ username: 'admin', message: 'Hello' }); expect(result).not.toHaveProperty('password'); }); });