chore: import upstream snapshot with attribution

test/redteam/shared/attackProviders.test.ts

@@ -0,0 +1,162 @@
+import { describe, expect, it } from 'vitest';
+import {
+  ATTACK_PROVIDER_IDS,
+  getAttackProviderFullId,
+  isAttackProvider,
+  isMultiTurnStrategy,
+} from '../../../src/redteam/shared/attackProviders';
+
+describe('attackProviders', () => {
+  describe('ATTACK_PROVIDER_IDS', () => {
+    it('should contain all expected multi-turn attack providers', () => {
+      expect(ATTACK_PROVIDER_IDS).toContain('hydra');
+      expect(ATTACK_PROVIDER_IDS).toContain('crescendo');
+      expect(ATTACK_PROVIDER_IDS).toContain('goat');
+      expect(ATTACK_PROVIDER_IDS).toContain('custom');
+    });
+
+    it('should contain all expected single-turn iterative attack providers', () => {
+      expect(ATTACK_PROVIDER_IDS).toContain('iterative');
+      expect(ATTACK_PROVIDER_IDS).toContain('iterative:meta');
+      expect(ATTACK_PROVIDER_IDS).toContain('iterative:tree');
+    });
+
+    it('should not contain unsupported strategies', () => {
+      expect(ATTACK_PROVIDER_IDS).not.toContain('mischievous-user');
+    });
+  });
+
+  describe('isAttackProvider', () => {
+    describe('multi-turn conversation providers', () => {
+      it('should return true for hydra variants', () => {
+        expect(isAttackProvider('hydra')).toBe(true);
+        expect(isAttackProvider('jailbreak:hydra')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:hydra')).toBe(true);
+      });
+
+      it('should return true for crescendo variants', () => {
+        expect(isAttackProvider('crescendo')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:crescendo')).toBe(true);
+      });
+
+      it('should return true for goat variants', () => {
+        expect(isAttackProvider('goat')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:goat')).toBe(true);
+      });
+
+      it('should return true for custom variants', () => {
+        expect(isAttackProvider('custom')).toBe(true);
+        expect(isAttackProvider('custom:my-custom')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:custom')).toBe(true);
+      });
+    });
+
+    describe('multi-attempt single-turn providers', () => {
+      it('should return true for iterative (base jailbreak)', () => {
+        expect(isAttackProvider('iterative')).toBe(true);
+        expect(isAttackProvider('jailbreak')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:iterative')).toBe(true);
+      });
+
+      it('should return true for iterative:meta (jailbreak:meta)', () => {
+        expect(isAttackProvider('iterative:meta')).toBe(true);
+        expect(isAttackProvider('jailbreak:meta')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:iterative:meta')).toBe(true);
+      });
+
+      it('should return true for iterative:tree (jailbreak:tree)', () => {
+        expect(isAttackProvider('iterative:tree')).toBe(true);
+        expect(isAttackProvider('jailbreak:tree')).toBe(true);
+        expect(isAttackProvider('promptfoo:redteam:iterative:tree')).toBe(true);
+      });
+    });
+
+    describe('non-attack providers', () => {
+      it('should return false for transform strategies', () => {
+        expect(isAttackProvider('base64')).toBe(false);
+        expect(isAttackProvider('rot13')).toBe(false);
+        expect(isAttackProvider('hex')).toBe(false);
+        expect(isAttackProvider('audio')).toBe(false);
+        expect(isAttackProvider('image')).toBe(false);
+      });
+
+      it('should return false for unsupported attack strategies', () => {
+        expect(isAttackProvider('mischievous-user')).toBe(false);
+      });
+
+      it('should return false for basic and other non-attack strategies', () => {
+        expect(isAttackProvider('basic')).toBe(false);
+        expect(isAttackProvider('default')).toBe(false);
+        expect(isAttackProvider('unknown')).toBe(false);
+      });
+    });
+  });
+
+  describe('getAttackProviderFullId', () => {
+    describe('already full IDs', () => {
+      it('should return the same ID if already full', () => {
+        expect(getAttackProviderFullId('promptfoo:redteam:hydra')).toBe('promptfoo:redteam:hydra');
+        expect(getAttackProviderFullId('promptfoo:redteam:iterative')).toBe(
+          'promptfoo:redteam:iterative',
+        );
+        expect(getAttackProviderFullId('promptfoo:redteam:iterative:meta')).toBe(
+          'promptfoo:redteam:iterative:meta',
+        );
+      });
+    });
+
+    describe('jailbreak variants', () => {
+      it('should convert jailbreak (base) to promptfoo:redteam:iterative', () => {
+        expect(getAttackProviderFullId('jailbreak')).toBe('promptfoo:redteam:iterative');
+      });
+
+      it('should convert jailbreak:meta to promptfoo:redteam:iterative:meta', () => {
+        expect(getAttackProviderFullId('jailbreak:meta')).toBe('promptfoo:redteam:iterative:meta');
+      });
+
+      it('should convert jailbreak:tree to promptfoo:redteam:iterative:tree', () => {
+        expect(getAttackProviderFullId('jailbreak:tree')).toBe('promptfoo:redteam:iterative:tree');
+      });
+
+      it('should convert jailbreak:hydra to promptfoo:redteam:hydra', () => {
+        expect(getAttackProviderFullId('jailbreak:hydra')).toBe('promptfoo:redteam:hydra');
+      });
+    });
+
+    describe('custom variants', () => {
+      it('should convert custom to promptfoo:redteam:custom', () => {
+        expect(getAttackProviderFullId('custom')).toBe('promptfoo:redteam:custom');
+      });
+
+      it('should convert custom:variant to promptfoo:redteam:custom', () => {
+        expect(getAttackProviderFullId('custom:my-custom')).toBe('promptfoo:redteam:custom');
+      });
+    });
+
+    describe('short IDs', () => {
+      it('should convert short IDs to full IDs', () => {
+        expect(getAttackProviderFullId('hydra')).toBe('promptfoo:redteam:hydra');
+        expect(getAttackProviderFullId('crescendo')).toBe('promptfoo:redteam:crescendo');
+        expect(getAttackProviderFullId('goat')).toBe('promptfoo:redteam:goat');
+      });
+    });
+  });
+
+  describe('isMultiTurnStrategy', () => {
+    it('should return true for multi-turn strategies', () => {
+      expect(isMultiTurnStrategy('hydra')).toBe(true);
+      expect(isMultiTurnStrategy('crescendo')).toBe(true);
+      expect(isMultiTurnStrategy('goat')).toBe(true);
+    });
+
+    it('should handle various ID formats', () => {
+      expect(isMultiTurnStrategy('jailbreak:hydra')).toBe(true);
+      expect(isMultiTurnStrategy('promptfoo:redteam:hydra')).toBe(true);
+    });
+
+    it('should return false for non-multi-turn strategies', () => {
+      expect(isMultiTurnStrategy('base64')).toBe(false);
+      expect(isMultiTurnStrategy('basic')).toBe(false);
+    });
+  });
+});

test/redteam/shared/runtimeTransform.test.ts

@@ -0,0 +1,250 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { applyRuntimeTransforms } from '../../../src/redteam/shared/runtimeTransform';
+
+import type { Strategy } from '../../../src/redteam/strategies/types';
+import type { TestCaseWithPlugin } from '../../../src/types';
+
+describe('runtimeTransform', () => {
+  const mockBase64Strategy: Strategy = {
+    id: 'base64',
+    action: vi.fn(async (testCases: TestCaseWithPlugin[]) =>
+      testCases.map((tc) => ({
+        ...tc,
+        vars: {
+          ...tc.vars,
+          input: Buffer.from(String(tc.vars?.input || '')).toString('base64'),
+        },
+      })),
+    ),
+  };
+
+  const mockAudioStrategy: Strategy = {
+    id: 'audio',
+    action: vi.fn(async (testCases: TestCaseWithPlugin[]) =>
+      testCases.map((tc) => ({
+        ...tc,
+        vars: {
+          ...tc.vars,
+          input: 'data:audio/mp3;base64,SGVsbG9BdWRpbw==', // Mock audio data URL
+        },
+      })),
+    ),
+  };
+
+  const mockImageStrategy: Strategy = {
+    id: 'image',
+    action: vi.fn(async (testCases: TestCaseWithPlugin[]) =>
+      testCases.map((tc) => ({
+        ...tc,
+        vars: {
+          ...tc.vars,
+          input: 'data:image/png;base64,SGVsbG9JbWFnZQ==', // Mock image data URL
+        },
+      })),
+    ),
+  };
+
+  const mockRawAudioStrategy: Strategy = {
+    id: 'audio-raw',
+    action: vi.fn(async (testCases: TestCaseWithPlugin[]) =>
+      testCases.map((tc) => ({
+        ...tc,
+        vars: {
+          ...tc.vars,
+          input: 'SGVsbG9SYXdBdWRpbw==', // Raw base64 without data URL
+        },
+      })),
+    ),
+  };
+
+  const mockStrategies: Strategy[] = [
+    mockBase64Strategy,
+    mockAudioStrategy,
+    mockImageStrategy,
+    mockRawAudioStrategy,
+  ];
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('applyRuntimeTransforms', () => {
+    it('should return original prompt when no layers provided', async () => {
+      const result = await applyRuntimeTransforms('test prompt', 'input', [], mockStrategies);
+
+      expect(result).toEqual({
+        prompt: 'test prompt',
+        originalPrompt: 'test prompt',
+      });
+    });
+
+    it('should return original prompt when layers is empty array', async () => {
+      const result = await applyRuntimeTransforms('test prompt', 'input', [], mockStrategies);
+
+      expect(result.prompt).toBe('test prompt');
+      expect(result.originalPrompt).toBe('test prompt');
+      expect(result.audio).toBeUndefined();
+      expect(result.image).toBeUndefined();
+    });
+
+    it('should apply single transform layer', async () => {
+      const result = await applyRuntimeTransforms('hello', 'input', ['base64'], mockStrategies);
+
+      expect(result.prompt).toBe('aGVsbG8='); // base64('hello')
+      expect(result.originalPrompt).toBe('hello');
+      expect(mockBase64Strategy.action).toHaveBeenCalledTimes(1);
+    });
+
+    it('should apply multiple transform layers in order', async () => {
+      // First base64, then the second strategy would transform the base64 result
+      // But for this test, let's just verify both are called
+      const mockSecondStrategy: Strategy = {
+        id: 'rot13',
+        action: vi.fn(async (testCases) => testCases),
+      };
+
+      const strategies = [...mockStrategies, mockSecondStrategy];
+
+      await applyRuntimeTransforms('hello', 'input', ['base64', 'rot13'], strategies);
+
+      expect(mockBase64Strategy.action).toHaveBeenCalledTimes(1);
+      expect(mockSecondStrategy.action).toHaveBeenCalledTimes(1);
+    });
+
+    it('should extract audio data from data URL', async () => {
+      const result = await applyRuntimeTransforms('hello', 'input', ['audio'], mockStrategies);
+
+      expect(result.audio).toEqual({
+        data: 'SGVsbG9BdWRpbw==',
+        format: 'mp3',
+      });
+      expect(result.originalPrompt).toBe('hello');
+    });
+
+    it('should extract image data from data URL', async () => {
+      const result = await applyRuntimeTransforms('hello', 'input', ['image'], mockStrategies);
+
+      expect(result.image).toEqual({
+        data: 'SGVsbG9JbWFnZQ==',
+        format: 'png',
+      });
+      expect(result.originalPrompt).toBe('hello');
+    });
+
+    it('should handle raw base64 audio (assume mp3 format)', async () => {
+      // Create a strategy that mimics returning raw base64 audio
+      const rawAudioStrategy: Strategy = {
+        id: 'audio',
+        action: vi.fn(async (testCases: TestCaseWithPlugin[]) =>
+          testCases.map((tc) => ({
+            ...tc,
+            vars: {
+              ...tc.vars,
+              input: 'SGVsbG9SYXdBdWRpbw==', // Raw base64 without data URL prefix
+            },
+          })),
+        ),
+      };
+
+      const strategies = [rawAudioStrategy];
+      const result = await applyRuntimeTransforms('hello', 'input', ['audio'], strategies);
+
+      expect(result.audio).toEqual({
+        data: 'SGVsbG9SYXdBdWRpbw==',
+        format: 'mp3', // Default format for raw base64
+      });
+    });
+
+    it('should handle layer config objects', async () => {
+      const result = await applyRuntimeTransforms(
+        'hello',
+        'input',
+        [{ id: 'base64', config: { someOption: true } }],
+        mockStrategies,
+      );
+
+      expect(result.prompt).toBe('aGVsbG8=');
+      expect(mockBase64Strategy.action).toHaveBeenCalledWith(
+        expect.any(Array),
+        'input',
+        expect.objectContaining({ someOption: true }),
+      );
+    });
+
+    it('should pass target context to per-turn layer strategies', async () => {
+      const result = await applyRuntimeTransforms('hello', 'input', ['base64'], mockStrategies, {
+        targetId: 'cloud-target-123',
+      });
+
+      expect(mockBase64Strategy.action).toHaveBeenCalledWith(
+        expect.any(Array),
+        'input',
+        expect.objectContaining({ targetId: 'cloud-target-123' }),
+      );
+
+      expect(result.prompt).toBe('aGVsbG8=');
+      expect(result.originalPrompt).toBe('hello');
+    });
+
+    it('should skip unknown strategies with warning', async () => {
+      const result = await applyRuntimeTransforms(
+        'hello',
+        'input',
+        ['unknown-strategy', 'base64'],
+        mockStrategies,
+      );
+
+      // Should still apply base64
+      expect(result.prompt).toBe('aGVsbG8=');
+      expect(mockBase64Strategy.action).toHaveBeenCalledTimes(1);
+    });
+
+    it('should return error when transform fails', async () => {
+      const failingStrategy: Strategy = {
+        id: 'failing',
+        action: vi.fn(async () => {
+          throw new Error('Transform failed');
+        }),
+      };
+
+      const strategies = [...mockStrategies, failingStrategy];
+      const result = await applyRuntimeTransforms('hello', 'input', ['failing'], strategies);
+
+      expect(result.error).toContain('Transform failing failed');
+      expect(result.prompt).toBe('hello'); // Returns original prompt on error
+      expect(result.originalPrompt).toBe('hello');
+    });
+
+    it('should preserve pluginId in metadata during transforms', async () => {
+      let capturedTestCase: TestCaseWithPlugin | undefined;
+
+      const inspectingStrategy: Strategy = {
+        id: 'inspect',
+        action: vi.fn(async (testCases: TestCaseWithPlugin[]) => {
+          capturedTestCase = testCases[0];
+          return testCases;
+        }),
+      };
+
+      const strategies = [...mockStrategies, inspectingStrategy];
+      await applyRuntimeTransforms('hello', 'input', ['inspect'], strategies);
+
+      expect(capturedTestCase?.metadata?.pluginId).toBe('runtime-transform');
+    });
+
+    it('should handle different inject variable names', async () => {
+      await applyRuntimeTransforms('test', 'query', ['base64'], mockStrategies);
+
+      // The strategy should receive the correct inject var name
+      expect(mockBase64Strategy.action).toHaveBeenCalledWith(
+        expect.arrayContaining([
+          expect.objectContaining({
+            vars: expect.objectContaining({ query: 'test' }),
+          }),
+        ]),
+        'query',
+        expect.any(Object),
+      );
+    });
+  });
+});