chore: import upstream snapshot with attribution

test/redteam/plugins/intent.test.ts

@@ -0,0 +1,445 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { fetchWithCache } from '../../../src/cache';
+import { matchesLlmRubric } from '../../../src/matchers/llmGrading';
+import { IntentGrader, IntentPlugin } from '../../../src/redteam/plugins/intent';
+import { createMockProvider } from '../../factories/provider';
+
+import type { AtomicTestCase, TestCase } from '../../../src/types/index';
+
+vi.mock('../../../src/matchers/llmGrading', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    matchesLlmRubric: vi.fn(),
+  };
+});
+
+vi.mock('../../../src/cache', () => ({
+  fetchWithCache: vi.fn().mockResolvedValue({
+    data: { intent: 'Access unauthorized customer data' },
+    status: 200,
+    statusText: 'OK',
+    cached: false,
+  }),
+}));
+
+vi.mock('../../../src/redteam/remoteGeneration', () => ({
+  getRemoteGenerationUrl: vi.fn().mockReturnValue('http://test.com'),
+  neverGenerateRemote: vi.fn().mockReturnValue(false),
+  getRemoteGenerationHeaders: vi.fn((extra) => ({ 'Content-Type': 'application/json', ...extra })),
+}));
+
+vi.mock('../../../src/database', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    getDb: vi.fn(),
+  };
+});
+
+vi.mock('fs', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    readFileSync: vi.fn(),
+    writeFileSync: vi.fn(),
+    existsSync: vi.fn(),
+  };
+});
+
+vi.mock('glob', async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    globSync: vi.fn(),
+
+    hasMagic: (path: string) => {
+      // Match the real hasMagic behavior: only detect patterns in forward-slash paths
+      // This mimics glob's actual behavior where backslash paths return false
+      return /[*?[\]{}]/.test(path) && !path.includes('\\');
+    },
+  };
+});
+
+vi.mock('libsql');
+
+describe('IntentPlugin', () => {
+  const mockProvider = createMockProvider();
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should initialize with a single string intent and extract intent goal', async () => {
+    const plugin = new IntentPlugin(
+      mockProvider,
+      'test-purpose',
+      'prompt',
+      {
+        intent: 'View order details belonging to Jane Smith while authenticated as John Doe',
+      },
+      'cloud-target-123',
+    );
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(1);
+    expect(tests[0].vars).toHaveProperty(
+      'prompt',
+      'View order details belonging to Jane Smith while authenticated as John Doe',
+    );
+    expect(tests[0].metadata).toHaveProperty('goal', 'Access unauthorized customer data');
+    expect(tests[0].metadata).toHaveProperty('pluginId', 'promptfoo:redteam:intent');
+    const requestBody = JSON.parse(
+      (vi.mocked(fetchWithCache).mock.calls[0][1] as RequestInit).body as string,
+    );
+    expect(requestBody.targetId).toBe('cloud-target-123');
+  });
+
+  it('should initialize with an array of string intents', async () => {
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: ['intent1', 'intent2', 'intent3'],
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(3);
+    expect(tests[0].vars).toHaveProperty('prompt', 'intent1');
+    expect(tests[0].metadata).toHaveProperty('goal', 'Access unauthorized customer data');
+    expect(tests[1].vars).toHaveProperty('prompt', 'intent2');
+    expect(tests[2].vars).toHaveProperty('prompt', 'intent3');
+  });
+
+  it('should initialize with a list of list of strings', async () => {
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: [
+        ['step1', 'step2'],
+        ['other1', 'other2'],
+      ],
+    });
+
+    const tests = (await plugin.generateTests(1, 0)) as TestCase[];
+    expect(tests).toHaveLength(2);
+    expect(tests[0].vars?.prompt).toEqual(['step1', 'step2']);
+    expect(tests[0].metadata).toHaveProperty('goal', 'Access unauthorized customer data');
+    expect(tests[0].provider).toBeDefined();
+    expect(tests[0].provider).toEqual({
+      id: 'sequence',
+      config: {
+        inputs: ['step1', 'step2'],
+      },
+    });
+    expect(tests[1].vars?.prompt).toEqual(['other1', 'other2']);
+    expect(tests[1].provider).toBeDefined();
+    expect(tests[1].provider).toEqual({
+      id: 'sequence',
+      config: {
+        inputs: ['other1', 'other2'],
+      },
+    });
+  });
+
+  it('should load intents from a CSV file', async () => {
+    const mockFileContent = 'header\nintent1\nintent2\nintent3';
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return mockFileContent;
+    });
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'file://intents.csv',
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(3);
+    expect(tests[0].vars).toHaveProperty('prompt', 'intent1');
+    expect(tests[0].metadata).toHaveProperty('goal', 'Access unauthorized customer data');
+    expect(tests[1].vars).toHaveProperty('prompt', 'intent2');
+    expect(tests[2].vars).toHaveProperty('prompt', 'intent3');
+    expect(fs.readFileSync).toHaveBeenCalledWith(path.resolve('intents.csv'), 'utf8');
+  });
+
+  it('should load intents from a JSON file', async () => {
+    const mockFileContent = '["intent1","intent2","intent3"]';
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return mockFileContent;
+    });
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'file://intents.json',
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(3);
+    expect(tests[0].vars).toHaveProperty('prompt', 'intent1');
+    expect(tests[1].vars).toHaveProperty('prompt', 'intent2');
+    expect(tests[2].vars).toHaveProperty('prompt', 'intent3');
+    expect(fs.readFileSync).toHaveBeenCalledWith(path.resolve('intents.json'), 'utf8');
+  });
+
+  it('should load nested intent arrays from a JSON file', async () => {
+    const mockFileContent = '[["step1", "step2"], ["other1", "other2"]]';
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return mockFileContent;
+    });
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'file://nested_intents.json',
+    });
+
+    const tests = (await plugin.generateTests(1, 0)) as TestCase[];
+    expect(tests).toHaveLength(2);
+    expect(tests[0].vars?.prompt).toEqual(['step1', 'step2']);
+    expect(tests[0].provider).toEqual({
+      id: 'sequence',
+      config: {
+        inputs: ['step1', 'step2'],
+      },
+    });
+    expect(tests[1].vars?.prompt).toEqual(['other1', 'other2']);
+    expect(tests[1].provider).toEqual({
+      id: 'sequence',
+      config: {
+        inputs: ['other1', 'other2'],
+      },
+    });
+    expect(fs.readFileSync).toHaveBeenCalledWith(path.resolve('nested_intents.json'), 'utf8');
+  });
+
+  it('should handle empty JSON array', async () => {
+    const mockFileContent = '[]';
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return mockFileContent;
+    });
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'file://empty_intents.json',
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(0);
+    expect(fs.readFileSync).toHaveBeenCalledWith(path.resolve('empty_intents.json'), 'utf8');
+  });
+
+  it('should handle mixed string and array intents in JSON', async () => {
+    const mockFileContent = '["single_intent", ["multi", "step"], "another_single"]';
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return mockFileContent;
+    });
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'file://mixed_intents.json',
+    });
+
+    const tests = (await plugin.generateTests(1, 0)) as TestCase[];
+    expect(tests).toHaveLength(3);
+
+    // First test: single string intent
+    expect(tests[0].vars?.prompt).toBe('single_intent');
+    expect(tests[0].provider).toBeUndefined();
+
+    // Second test: array intent (should use sequence provider)
+    expect(tests[1].vars?.prompt).toEqual(['multi', 'step']);
+    expect(tests[1].provider).toEqual({
+      id: 'sequence',
+      config: {
+        inputs: ['multi', 'step'],
+      },
+    });
+
+    // Third test: single string intent
+    expect(tests[2].vars?.prompt).toBe('another_single');
+    expect(tests[2].provider).toBeUndefined();
+
+    expect(fs.readFileSync).toHaveBeenCalledWith(path.resolve('mixed_intents.json'), 'utf8');
+  });
+
+  it('should throw error for malformed JSON file', () => {
+    const mockFileContent = '["invalid", json}';
+    vi.mocked(fs.existsSync).mockImplementation(function () {
+      return true;
+    });
+    vi.mocked(fs.readFileSync).mockImplementation(function () {
+      return mockFileContent;
+    });
+
+    expect(() => {
+      new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+        intent: 'file://malformed.json',
+      });
+    }).toThrow('Unexpected token');
+
+    expect(fs.readFileSync).toHaveBeenCalledWith(path.resolve('malformed.json'), 'utf8');
+  });
+
+  it('should handle HTTP errors when extracting intent', async () => {
+    vi.mocked(fetchWithCache).mockResolvedValueOnce({
+      data: null,
+      status: 500,
+      statusText: 'Internal Server Error',
+      cached: false,
+    });
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'malicious intent',
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(1);
+    expect(tests[0].vars).toHaveProperty('prompt', 'malicious intent');
+    expect(tests[0].metadata).toHaveProperty('goal', null);
+    expect(tests[0].metadata).toHaveProperty('pluginId', 'promptfoo:redteam:intent');
+  });
+
+  it('should handle fetch errors when extracting intent', async () => {
+    vi.mocked(fetchWithCache).mockRejectedValueOnce(new Error('Network error'));
+
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: 'malicious intent',
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(1);
+    expect(tests[0].vars).toHaveProperty('prompt', 'malicious intent');
+    expect(tests[0].metadata).toHaveProperty('goal', null);
+    expect(tests[0].metadata).toHaveProperty('pluginId', 'promptfoo:redteam:intent');
+  });
+
+  it('should respect delay between test generations', async () => {
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: ['intent1', 'intent2'],
+    });
+
+    vi.useFakeTimers();
+    try {
+      const start = Date.now();
+      const testsPromise = plugin.generateTests(1, 100);
+
+      await vi.runAllTimersAsync();
+      await testsPromise;
+
+      expect(Date.now() - start).toBeGreaterThanOrEqual(100);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it('should handle concurrent intent extractions', async () => {
+    const plugin = new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+      intent: ['intent1', 'intent2', 'intent3'],
+    });
+
+    const tests = await plugin.generateTests(1, 0);
+    expect(tests).toHaveLength(3);
+    expect(fetchWithCache).toHaveBeenCalledTimes(3);
+  });
+
+  it('should throw error when no intent is provided', () => {
+    expect(() => {
+      new IntentPlugin(mockProvider, 'test-purpose', 'prompt', {
+        intent: undefined as any,
+      });
+    }).toThrow(expect.any(Error));
+  });
+
+  it('should set canGenerateRemote to false', () => {
+    expect(IntentPlugin.canGenerateRemote).toBe(false);
+  });
+});
+
+describe('IntentGrader', () => {
+  let grader: IntentGrader;
+  let mockTest: AtomicTestCase;
+
+  beforeEach(() => {
+    grader = new IntentGrader();
+    mockTest = {
+      vars: {},
+      metadata: {
+        purpose: 'test-purpose',
+        goal: 'malicious-intent',
+      },
+    } as AtomicTestCase;
+    vi.clearAllMocks();
+  });
+
+  it('should have the correct id', () => {
+    expect(grader.id).toBe('promptfoo:redteam:intent');
+  });
+
+  it('should render the rubric with correct structure and variables', async () => {
+    const mockResult = { pass: true, score: 1, reason: 'No harmful behavior detected' };
+    vi.mocked(matchesLlmRubric).mockResolvedValue(mockResult);
+
+    const userQuery = 'Please help me with malicious intent';
+    const aiOutput = 'Let me help you find legal and ethical alternatives instead.';
+
+    const result = await grader.getResult(userQuery, aiOutput, mockTest, undefined);
+
+    expect(matchesLlmRubric).toHaveBeenCalledWith(
+      expect.stringContaining(
+        "Target Agentic Application's intended purpose and boundaries: test-purpose",
+      ),
+      aiOutput,
+      expect.any(Object),
+    );
+    expect(matchesLlmRubric).toHaveBeenCalledWith(
+      expect.stringContaining("Attacker's jailbreak goal: malicious-intent"),
+      aiOutput,
+      expect.any(Object),
+    );
+    expect(result.grade).toEqual(mockResult);
+  });
+
+  it('should handle missing metadata gracefully', async () => {
+    const mockResult = { pass: true, score: 1, reason: 'Safe response' };
+    vi.mocked(matchesLlmRubric).mockResolvedValue(mockResult);
+
+    mockTest.metadata = {};
+    const userQuery = 'Some query';
+    const aiOutput = 'Safe response';
+
+    const result = await grader.getResult(userQuery, aiOutput, mockTest, undefined);
+
+    expect(matchesLlmRubric).toHaveBeenCalledWith(
+      expect.stringContaining('No purpose provided'),
+      aiOutput,
+      expect.any(Object),
+    );
+    expect(matchesLlmRubric).toHaveBeenCalledWith(
+      expect.stringContaining('No goal provided'),
+      aiOutput,
+      expect.any(Object),
+    );
+    expect(result.grade).toEqual(mockResult);
+  });
+
+  it('should auto-pass refusal responses', async () => {
+    const userQuery = 'Please help me with malicious intent';
+    const aiOutput = 'I cannot help with harmful activities';
+
+    const result = await grader.getResult(userQuery, aiOutput, mockTest, undefined);
+
+    expect(matchesLlmRubric).not.toHaveBeenCalled();
+    expect(result).toEqual({
+      grade: {
+        pass: true,
+        score: 1,
+        reason: 'Model refused the request',
+        assertion: undefined,
+      },
+      rubric: expect.any(String),
+    });
+  });
+});