import * as fs from 'fs';

import cliProgress from 'cli-progress';
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import logger from '../../src/logger';
import { loadApiProvider } from '../../src/providers/index';
import {
  getDefaultNFanout,
  HARM_PLUGINS,
  MULTI_INPUT_VAR,
  PII_PLUGINS,
} from '../../src/redteam/constants';
import { extractEntities } from '../../src/redteam/extraction/entities';
import { extractSystemPurpose } from '../../src/redteam/extraction/purpose';
import {
  calculateTotalTests,
  getTestCount,
  resolvePluginConfig,
  synthesize,
} from '../../src/redteam/index';
import { Plugins } from '../../src/redteam/plugins/index';
import { getRemoteHealthUrl, shouldGenerateRemote } from '../../src/redteam/remoteGeneration';
import { Strategies, validateStrategies } from '../../src/redteam/strategies/index';
import { checkRemoteHealth } from '../../src/util/apiHealth';
import { extractVariablesFromTemplates } from '../../src/util/templates';
import { loadYaml } from '../../src/util/yamlLoad';
import { mockProcessEnv, stripAnsi } from '../util/utils';

import type { Inputs } from '../../src/types/shared';

vi.mock('cli-progress');
vi.mock('../../src/logger');
vi.mock('../../src/providers');
vi.mock('../../src/redteam/extraction/entities');
vi.mock('../../src/redteam/extraction/purpose');
vi.mock('../../src/util/templates', async () => {
  const originalModule = await vi.importActual('../../src/util/templates');
  return {
    ...originalModule,
    extractVariablesFromTemplates: vi.fn().mockReturnValue(['query']),
  };
});

vi.mock('../../src/redteam/strategies', async () => ({
  ...(await vi.importActual('../../src/redteam/strategies')),

  validateStrategies: vi.fn().mockImplementation(function (strategies) {
    if (strategies.some((s: { id: string }) => s.id === 'invalid-strategy')) {
      throw new Error('Invalid strategies');
    }
  }),
}));

vi.mock('../../src/util/apiHealth');
vi.mock('../../src/redteam/remoteGeneration');
vi.mock('../../src/redteam/sharpAvailability', async () => ({
  ...(await vi.importActual('../../src/redteam/sharpAvailability')),
  validateSharpDependency: vi.fn().mockResolvedValue(undefined),
}));
vi.mock('../../src/redteam/util', async () => ({
  ...(await vi.importActual('../../src/redteam/util')),
  extractGoalFromPrompt: vi.fn().mockResolvedValue('mocked goal'),
}));

describe('synthesize', () => {
  const mockProvider = {
    callApi: vi.fn(),
    generate: vi.fn(),
    id: () => 'test-provider',
  };

  afterAll(() => {
    vi.restoreAllMocks();
  });

  beforeEach(() => {
    vi.resetAllMocks();

    // Set up logger mocks
    vi.mocked(logger.info).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.warn).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.error).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.debug).mockImplementation(function () {
      return logger as any;
    });

    // Set up templates mock with consistent default behavior
    vi.mocked(extractVariablesFromTemplates).mockImplementation(function () {
      return ['query'];
    });

    vi.mocked(extractEntities).mockResolvedValue(['entity1', 'entity2']);
    vi.mocked(extractSystemPurpose).mockResolvedValue('Test purpose');
    vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
    vi.spyOn(process, 'exit').mockImplementation(function (
      code?: string | number | null | undefined,
    ) {
      throw new Error(`Process.exit called with code ${code}`);
    });
    vi.mocked(validateStrategies).mockImplementation(async function () {});
    vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
      return {
        increment: vi.fn(),
        start: vi.fn(),
        stop: vi.fn(),
        update: vi.fn(),
      } as any;
    });
    // Disable remote generation by default to avoid health checks interfering
    // with tests that don't explicitly set this behaviour
    vi.mocked(shouldGenerateRemote).mockImplementation(function () {
      return false;
    });
    vi.mocked(getRemoteHealthUrl).mockImplementation(function () {
      return 'https://api.test/health';
    });
    vi.mocked(checkRemoteHealth).mockResolvedValue({
      status: 'OK',
      message: 'Cloud API is healthy',
    });
  });

  // Input handling tests
  describe('Input handling', () => {
    it('should use provided purpose and entities if given', async () => {
      const result = await synthesize({
        entities: ['custom-entity'],
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        purpose: 'Custom purpose',
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(result).toEqual(
        expect.objectContaining({
          entities: ['custom-entity'],
          purpose: 'Custom purpose',
        }),
      );
      expect(extractEntities).not.toHaveBeenCalled();
      expect(extractSystemPurpose).not.toHaveBeenCalled();
    });

    it('should pass resolved target context when extracting purpose and entities', async () => {
      const generationContext = {
        providerTargetIds: ['file://local-provider.ts'],
        cloudTargetId: 'cloud-target-123',
      };
      await synthesize({
        cloudTargetDatabaseId: 'cloud-target-123',
        language: 'english',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['file://local-provider.ts'],
      });

      expect(extractEntities).toHaveBeenCalledWith(
        expect.any(Object),
        ['Test prompt'],
        generationContext,
      );
      expect(extractSystemPurpose).toHaveBeenCalledWith(
        expect.any(Object),
        ['Test prompt'],
        generationContext,
      );
    });

    it('should handle empty prompts array', async () => {
      await expect(
        synthesize({
          language: 'en',
          numTests: 1,
          plugins: [{ id: 'test-plugin', numTests: 1 }],
          prompts: [] as unknown as [string, ...string[]],
          strategies: [],
          targetIds: ['test-provider'],
        }),
      ).rejects.toThrow('Prompts array cannot be empty');
    });

    it('should correctly process multiple prompts', async () => {
      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Prompt 1', 'Prompt 2', 'Prompt 3'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(extractSystemPurpose).toHaveBeenCalledWith(
        expect.any(Object),
        ['Prompt 1', 'Prompt 2', 'Prompt 3'],
        expect.any(Object),
      );
      expect(extractEntities).toHaveBeenCalledWith(
        expect.objectContaining({ id: expect.any(Function) }),
        ['Prompt 1', 'Prompt 2', 'Prompt 3'],
        expect.any(Object),
      );
    });
  });

  // API provider tests
  describe('API provider', () => {
    it('should use the provided API provider if given', async () => {
      const customProvider = {
        callApi: vi.fn(),
        generate: vi.fn(),
        id: () => 'custom-provider',
      };
      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        provider: customProvider,
        strategies: [],
        targetIds: ['custom-provider'],
      });

      expect(loadApiProvider).not.toHaveBeenCalled();
    });
  });

  // Plugin and strategy tests
  describe('Plugins and strategies', () => {
    it('should generate test cases for each plugin', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'plugin1', numTests: 2 },
          { id: 'plugin2', numTests: 3 },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(mockPluginAction).toHaveBeenCalledTimes(2);
      expect(result.testCases).toEqual([
        expect.objectContaining({ metadata: expect.objectContaining({ pluginId: 'plugin1' }) }),
        expect.objectContaining({ metadata: expect.objectContaining({ pluginId: 'plugin2' }) }),
      ]);
    });

    it('should pass maxCharsPerMessage through synthesize into plugin metadata and strategy config', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'short' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((testCase: any) => ({
          ...testCase,
          metadata: {
            ...testCase.metadata,
            strategyId: 'goat',
          },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        action: mockStrategyAction,
        id: 'goat',
      });

      const result = await synthesize({
        maxCharsPerMessage: 12,
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'goat' }],
        targetIds: ['test-provider'],
      });

      expect(mockPluginAction).toHaveBeenCalledWith(
        expect.objectContaining({
          config: expect.objectContaining({
            maxCharsPerMessage: 12,
            modifiers: expect.objectContaining({
              maxCharsPerMessage: 'Each generated user message must be 12 characters or fewer.',
            }),
          }),
        }),
      );
      expect(mockStrategyAction).toHaveBeenCalledWith(
        expect.any(Array),
        'query',
        expect.objectContaining({
          maxCharsPerMessage: 12,
        }),
        'goat',
      );
      expect(result.testCases).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            metadata: expect.objectContaining({
              pluginId: 'test-plugin',
              pluginConfig: expect.objectContaining({
                maxCharsPerMessage: 12,
              }),
              modifiers: expect.objectContaining({
                maxCharsPerMessage: 'Each generated user message must be 12 characters or fewer.',
              }),
            }),
          }),
          expect.objectContaining({
            metadata: expect.objectContaining({
              strategyConfig: expect.objectContaining({
                maxCharsPerMessage: 12,
              }),
            }),
          }),
        ]),
      );
    });

    it('should pass shared generation options through custom file plugins', async () => {
      vi.spyOn(Plugins, 'find').mockReturnValue(undefined);
      const pluginPath = 'test/redteam/fixtures/custom-plugin-shared-options.yaml';
      const fileUtil = await import('../../src/util/file');
      vi.spyOn(fileUtil, 'maybeLoadFromExternalFile').mockImplementation((filePath) => {
        if (String(filePath).endsWith(pluginPath)) {
          return {
            id: 'stable-shared-options',
            generator: 'Prompt: shared option probe',
            grader: 'Grade the response based on {{ purpose }}',
            metric: 'custom-shared-options',
          };
        }
        return filePath;
      });

      mockProvider.callApi.mockImplementation(async (input: string) => ({
        output: input.includes('language: fr')
          ? 'Prompt: bonjour'
          : input.includes('language: de')
            ? 'Prompt: guten tag'
            : 'Prompt: fallback',
      }));

      const result = await synthesize({
        language: 'en',
        maxCharsPerMessage: 50,
        numTests: 1,
        plugins: [
          {
            id: `file://./${pluginPath}`,
            numTests: 1,
            config: {
              language: ['fr', 'de'],
              modifiers: { tone: 'formal' },
            },
            severity: 'high',
          },
        ],
        prompts: ['Test prompt'],
        provider: mockProvider,
        purpose: 'Custom plugin purpose',
        strategies: [],
        targetIds: ['test-provider'],
        testGenerationInstructions: 'Prefer edge-case probes',
      });

      expect(mockProvider.callApi).toHaveBeenCalledTimes(2);
      for (const call of mockProvider.callApi.mock.calls) {
        expect(call[0]).toContain('testGenerationInstructions: Prefer edge-case probes');
        expect(call[0]).toContain('tone: formal');
        expect(call[0]).toContain(
          'maxCharsPerMessage: Each generated user message must be 50 characters or fewer.',
        );
      }
      expect(mockProvider.callApi.mock.calls.map((call) => call[0])).toEqual(
        expect.arrayContaining([
          expect.stringContaining('language: fr'),
          expect.stringContaining('language: de'),
        ]),
      );
      expect(result.testCases).toHaveLength(2);
      expect(result.testCases.map((testCase) => testCase.metadata?.language).sort()).toEqual([
        'de',
        'fr',
      ]);
      expect(result.testCases).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            metadata: expect.objectContaining({
              pluginId: `file://./${pluginPath}`,
              pluginConfig: expect.objectContaining({
                language: expect.stringMatching(/^(fr|de)$/),
                maxCharsPerMessage: 50,
              }),
              severity: 'high',
              modifiers: expect.objectContaining({
                language: expect.stringMatching(/^(fr|de)$/),
                maxCharsPerMessage: 'Each generated user message must be 50 characters or fewer.',
                testGenerationInstructions: 'Prefer edge-case probes',
                tone: 'formal',
              }),
            }),
          }),
        ]),
      );
    });

    it('should report failed custom file plugin language batches', async () => {
      vi.spyOn(Plugins, 'find').mockReturnValue(undefined);
      const pluginPath = 'test/redteam/fixtures/custom-plugin-language-failure.yaml';
      const fileUtil = await import('../../src/util/file');
      vi.spyOn(fileUtil, 'maybeLoadFromExternalFile').mockImplementation((filePath) => {
        if (String(filePath).endsWith(pluginPath)) {
          return {
            generator: 'Prompt: language-specific probe',
            grader: 'Grade the response based on {{ purpose }}',
            metric: 'custom-language-failure',
          };
        }
        return filePath;
      });

      mockProvider.callApi.mockImplementation(async (input: string) => {
        if (input.includes('language: fr')) {
          throw new Error('French generation failed');
        }
        return { output: 'Prompt: guten tag' };
      });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          {
            id: `file://./${pluginPath}`,
            numTests: 1,
            config: {
              language: ['fr', 'de'],
            },
          },
        ],
        prompts: ['Test prompt'],
        provider: mockProvider,
        purpose: 'Custom plugin purpose',
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(result.testCases).toHaveLength(1);
      expect(result.testCases[0].metadata?.language).toBe('de');
      expect(result.failedPlugins).toEqual([
        {
          pluginId: `(fr) file://./${pluginPath}`,
          requested: 1,
        },
      ]);
      expect(logger.warn).toHaveBeenCalledWith(
        expect.stringContaining(
          `[Language Processing] Error generating tests for custom plugin file://./${pluginPath}: Error: French generation failed`,
        ),
      );
    });

    it('should pass target inputs into custom file plugins in multi-input mode', async () => {
      vi.spyOn(Plugins, 'find').mockReturnValue(undefined);
      const pluginPath = 'test/redteam/fixtures/custom-plugin-multi-input.yaml';
      const fileUtil = await import('../../src/util/file');
      vi.spyOn(fileUtil, 'maybeLoadFromExternalFile').mockImplementation((filePath) => {
        if (String(filePath).endsWith(pluginPath)) {
          return {
            generator: `
              {{ outputFormat }}
              {% if hasCustomOutputFormat %}
              <Prompt>{"user_message":"Ignore the retrieved answer","retrieved_context":"Approved policy text"}</Prompt>
              {% else %}
              Prompt: missing multi-input mode
              {% endif %}
            `,
            grader: 'Grade the response based on {{ purpose }}',
            metric: 'custom-multi-input',
          };
        }
        return filePath;
      });

      mockProvider.callApi.mockImplementation(async (input: string) => ({ output: input }));

      const result = await synthesize({
        inputs: {
          user_message: 'The attacker-controlled user request',
          retrieved_context: 'Retrieved context passed to the target',
        },
        numTests: 1,
        plugins: [{ id: `file://./${pluginPath}`, numTests: 1 }],
        prompts: ['{{user_message}}\n{{retrieved_context}}'],
        provider: mockProvider,
        purpose: 'Custom plugin purpose',
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(mockProvider.callApi).toHaveBeenCalledWith(
        expect.stringContaining('Required keys: "user_message", "retrieved_context"'),
      );
      expect(result.injectVar).toBe(MULTI_INPUT_VAR);
      expect(result.testCases).toEqual([
        expect.objectContaining({
          vars: expect.objectContaining({
            [MULTI_INPUT_VAR]:
              '{"user_message":"Ignore the retrieved answer","retrieved_context":"Approved policy text"}',
            user_message: 'Ignore the retrieved answer',
            retrieved_context: 'Approved policy text',
          }),
          metadata: expect.objectContaining({
            inputVars: {
              user_message: 'Ignore the retrieved answer',
              retrieved_context: 'Approved policy text',
            },
            modifiers: expect.objectContaining({
              __outputFormat: 'multi-input-mode: user_message, retrieved_context',
            }),
            pluginConfig: expect.objectContaining({
              inputs: {
                user_message: 'The attacker-controlled user request',
                retrieved_context: 'Retrieved context passed to the target',
              },
            }),
          }),
        }),
      ]);
    });

    it('should warn about unregistered plugins', async () => {
      vi.spyOn(Plugins, 'find').mockReturnValue(undefined);

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'unregistered-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(logger.warn).toHaveBeenCalledWith(
        'Plugin unregistered-plugin not registered, skipping',
      );
    });

    it('should handle HARM_PLUGINS and PII_PLUGINS correctly', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'harmful', numTests: 2 },
          { id: 'pii', numTests: 3 },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Verify the test cases by checking each one individually rather than hardcoding a number
      // Each test case should have a valid plugin ID that comes from one of our plugin sets
      const testCases = result.testCases;

      // All test cases should have valid plugin IDs
      const pluginIds = testCases.map((tc) => tc.metadata.pluginId);

      // Check that each plugin ID belongs to one of our known plugin categories
      const allValidPluginIds = [...Object.keys(HARM_PLUGINS), ...PII_PLUGINS];

      // Every plugin ID should be in our list of valid plugins
      pluginIds.forEach((id) => {
        expect(allValidPluginIds).toContain(id);
      });

      // Check for uniqueness - we should have unique plugin IDs (no duplicates of the same plugin)
      const uniquePluginIds = new Set(pluginIds);

      // The expected number of test cases is the number of unique plugin IDs we actually got
      // This is more reliable than trying to predict the exact expansion logic
      const expectedTestCount = uniquePluginIds.size;

      // Assert that we got exactly the expected number of test cases
      expect(testCases).toHaveLength(expectedTestCount);
    });

    it('should generate a correct report for plugins and strategies', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockStrategyAction = vi.fn().mockReturnValue([{ test: 'strategy case' }]);
      vi.spyOn(Strategies, 'find').mockReturnValue({
        action: mockStrategyAction,
        id: 'mockStrategy',
      });

      await synthesize({
        language: 'en',
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'mockStrategy' }],
        targetIds: ['test-provider'],
      });

      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('Test Generation Report:'));
      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('test-plugin'));
      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('mockStrategy'));
    });

    it('should use default fan-out values when strategy config omits n', async () => {
      const pluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      const pluginFindSpy = vi
        .spyOn(Plugins, 'find')
        .mockReturnValue({ action: pluginAction, key: 'mockPlugin' });

      const strategyAction = vi.fn().mockImplementation(async function () {
        return Array.from({ length: 5 }, (_, idx) => ({ vars: { query: `fanout-${idx}` } }));
      });
      const strategyFindSpy = vi.spyOn(Strategies, 'find').mockImplementation(function (predicate) {
        if (typeof predicate === 'function') {
          const strategies = [{ action: strategyAction, id: 'jailbreak:composite' }];
          return strategies.find(predicate);
        }
        return undefined;
      });

      try {
        await synthesize({
          language: 'en',
          numTests: 1,
          plugins: [{ id: 'test-plugin', numTests: 1 }],
          prompts: ['Test prompt'],
          strategies: [{ id: 'jailbreak:composite' }],
          targetIds: ['test-provider'],
        });

        const reportMessage = vi
          .mocked(logger.info)
          .mock.calls.map(([arg]) => arg)
          .find(
            (arg): arg is string =>
              typeof arg === 'string' && arg.includes('Test Generation Report'),
          );

        expect(reportMessage).toBeDefined();

        const stripped = stripAnsi(reportMessage!);
        const lines = stripped.split('\n');

        // Find header row and parse column indexes
        const headerRow = lines.find(
          (line) => line.includes('Requested') && line.includes('Generated'),
        );
        expect(headerRow).toBeDefined();

        const headerColumns = headerRow!
          .split('│')
          .map((col) => col.trim())
          .filter((col) => col.length > 0);

        const requestedIndex = headerColumns.indexOf('Requested');
        const generatedIndex = headerColumns.indexOf('Generated');
        expect(requestedIndex).toBeGreaterThan(-1);
        expect(generatedIndex).toBeGreaterThan(-1);

        // Find the jailbreak:composite row
        const row = lines.find((line) => line.includes('jailbreak:composite'));
        expect(row).toBeDefined();

        const columns = row!
          .split('│')
          .map((col) => col.trim())
          .filter((col) => col.length > 0);

        // Requested = default fan-out * 1 base test
        expect(columns[requestedIndex]).toBe(getDefaultNFanout('jailbreak:composite').toString());
        // Generated = what our mock strategy returned
        expect(columns[generatedIndex]).toBe(getDefaultNFanout('jailbreak:composite').toString());

        // Verify the "Using strategies:" summary shows correct fan-out
        const summaryMessage = vi
          .mocked(logger.info)
          .mock.calls.map(([arg]) => arg)
          .find(
            (arg): arg is string => typeof arg === 'string' && arg.includes('Using strategies:'),
          );

        expect(summaryMessage).toBeDefined();
        const summaryStripped = stripAnsi(summaryMessage!);
        const summaryLine = summaryStripped
          .split('\n')
          .find((line) => line.includes('jailbreak:composite'));
        expect(summaryLine).toBeDefined();
        expect(summaryLine).toContain(
          `(${getDefaultNFanout('jailbreak:composite')} additional tests)`,
        );

        // Verify the "Test Generation Summary:" shows correct total
        const totalSummaryMessage = vi
          .mocked(logger.info)
          .mock.calls.map(([arg]) => arg)
          .find(
            (arg): arg is string =>
              typeof arg === 'string' && arg.includes('Test Generation Summary:'),
          );

        expect(totalSummaryMessage).toBeDefined();
        const totalStripped = stripAnsi(totalSummaryMessage!);
        // 1 base + default fan-out
        expect(totalStripped).toContain(
          `• Total tests: ${1 + getDefaultNFanout('jailbreak:composite')}`,
        );
      } finally {
        pluginFindSpy.mockRestore();
        strategyFindSpy.mockRestore();
      }
    });

    it('should expand strategy collections into individual strategies', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockStrategyAction = vi.fn().mockReturnValue([{ test: 'strategy case' }]);
      vi.spyOn(Strategies, 'find').mockImplementation(function (s: any) {
        if (['morse', 'piglatin'].includes(s.id)) {
          return { action: mockStrategyAction, id: s.id };
        }
        return undefined;
      });

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [
          {
            id: 'other-encodings',
            config: { customOption: 'test-value' },
          },
        ],
        targetIds: ['test-provider'],
      });

      expect(validateStrategies).toHaveBeenCalledWith(expect.any(Array));
    });

    it('should deduplicate strategies with the same ID', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      // Create a spy on validateStrategies to capture the strategies array
      const validateStrategiesSpy = vi.mocked(validateStrategies);
      validateStrategiesSpy.mockClear();

      // Include both the collection and an individual strategy that's part of the collection
      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [
          { id: 'other-encodings' },
          { id: 'morse' }, // This is already included in other-encodings
        ],
        targetIds: ['test-provider'],
      });

      // Check that validateStrategies was called
      expect(validateStrategiesSpy).toHaveBeenCalledWith(expect.any(Array));

      // Look at the strategies that were passed to validateStrategies
      // The array should have no duplicate ids
      const strategiesArg = validateStrategiesSpy.mock.calls[0][0];
      const strategyIds = strategiesArg.map((s: any) => s.id);

      // Check for duplicates
      const uniqueIds = new Set(strategyIds);
      expect(uniqueIds.size).toBe(strategyIds.length);

      // Should have morse only once
      expect(strategyIds.filter((id: string) => id === 'morse')).toHaveLength(1);

      // Should have at least morse and piglatin
      expect(strategyIds).toContain('morse');
      expect(strategyIds).toContain('piglatin');
    });

    it('should handle missing strategy collections gracefully', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [
          { id: 'unknown-collection' }, // This doesn't exist in the mappings
        ],
        targetIds: ['test-provider'],
      });

      expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('unknown-collection'));
    });

    it('should find exact strategy IDs like jailbreak:composite', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockJailbreakCompositeAction = vi.fn().mockReturnValue([
        {
          vars: { query: 'composite test' },
          metadata: { strategyId: 'jailbreak:composite' },
        },
      ]);

      // Mock the Strategies array to include both jailbreak and jailbreak:composite
      vi.spyOn(Strategies, 'find').mockImplementation(function (predicate) {
        if (typeof predicate === 'function') {
          const strategies = [
            {
              id: 'jailbreak',
              action: vi.fn().mockReturnValue([{ vars: { query: 'basic jailbreak' } }]),
            },
            { id: 'jailbreak:composite', action: mockJailbreakCompositeAction },
          ];
          return strategies.find(predicate);
        }
        return undefined;
      });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak:composite' }],
        targetIds: ['test-provider'],
      });

      // Should have called the composite action, not the basic jailbreak action
      expect(mockJailbreakCompositeAction).toHaveBeenCalled();

      // Check that the strategy test cases have the correct strategy ID
      const strategyTestCases = result.testCases.filter((tc) => tc.metadata?.strategyId);
      expect(strategyTestCases).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            metadata: expect.objectContaining({ strategyId: 'jailbreak:composite' }),
          }),
        ]),
      );
    });

    it('should re-materialize DOCX inputs after strategies mutate multi-input prompts', async () => {
      const inputs = {
        document: {
          config: {
            injectionPlacements: ['comment'],
            inputPurpose: 'Summarize an uploaded policy draft',
          },
          description: 'DOCX document to summarize',
          type: 'docx',
        },
        question: {
          config: {
            benign: true,
          },
          description: 'Benign user question',
          type: 'text',
        },
      } satisfies Inputs;
      const stalePayload = 'stale pre-strategy payload';
      const strategyPayload = 'strategy-mutated payload';
      const strategyQuestion = 'Please summarize the uploaded document.';
      const wrapperSummary = 'Fresh strategy wrapper summary';
      const injectedInstruction = 'Fresh strategy injected instruction';
      const baseMultiInputPrompt = JSON.stringify({
        document: stalePayload,
        question: 'base question',
      });
      const strategyMultiInputPrompt = JSON.stringify({
        document: strategyPayload,
        question: strategyQuestion,
      });

      mockProvider.callApi.mockResolvedValue({
        output: JSON.stringify({
          bodyText: 'Fresh strategy document body',
          injectedInstruction,
          injectionPlacement: 'comment',
          wrapperSummary,
        }),
      });

      const mockPluginAction = vi.fn().mockResolvedValue([
        {
          metadata: {
            inputMaterialization: {
              document: {
                injectedInstruction: 'stale injected instruction',
                injectionPlacement: 'body',
                inputPurpose: 'Summarize an uploaded policy draft',
                wrapperSummary: 'stale wrapper summary',
              },
            },
            pluginConfig: { inputs },
            pluginId: 'prompt-extraction',
          },
          vars: {
            [MULTI_INPUT_VAR]: baseMultiInputPrompt,
            document: 'stale-document-data-uri',
            question: 'base question',
          },
        },
      ]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'prompt-extraction',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases: any[]) =>
        testCases.map((testCase: any) => ({
          ...testCase,
          vars: {
            ...testCase.vars,
            [MULTI_INPUT_VAR]: strategyMultiInputPrompt,
          },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        action: mockStrategyAction,
        id: 'jailbreak:meta',
      });

      const result = await synthesize({
        inputs,
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'prompt-extraction', numTests: 1 }],
        prompts: ['{{document}} {{question}}'],
        provider: mockProvider,
        purpose: 'Summarize uploaded documents',
        strategies: [{ id: 'jailbreak:meta' }],
        targetIds: ['test-provider'],
      });

      const strategyTestCase = result.testCases.find(
        (testCase) => testCase.metadata?.strategyId === 'jailbreak:meta',
      );

      expect(strategyTestCase).toBeDefined();
      expect(mockProvider.callApi).toHaveBeenCalledWith(expect.stringContaining(strategyPayload));
      expect(mockProvider.callApi).not.toHaveBeenCalledWith(expect.stringContaining(stalePayload));
      expect(strategyTestCase?.vars?.document).toEqual(
        expect.stringMatching(
          /^data:application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document;base64,/,
        ),
      );
      expect(strategyTestCase?.vars?.question).toBe(strategyQuestion);
      expect(strategyTestCase?.metadata?.inputMaterialization).toMatchObject({
        document: {
          injectedInstruction,
          injectionPlacement: 'comment',
          inputPurpose: 'Summarize an uploaded policy draft',
          wrapperSummary,
        },
      });
    });

    it('should preserve existing DOCX materialization when strategies leave the multi-input prompt unchanged', async () => {
      const inputs = {
        document: {
          config: {
            benign: false,
            injectionPlacements: ['comment'],
            inputPurpose: 'Summarize an uploaded policy draft',
          },
          description: 'DOCX document to summarize',
          type: 'docx',
        },
        question: {
          config: {
            benign: true,
          },
          description: 'Benign user question',
          type: 'text',
        },
      } satisfies Inputs;
      const baseMultiInputPrompt = JSON.stringify({
        document: 'stale pre-strategy payload',
        question: 'base question',
      });

      const mockPluginAction = vi.fn().mockResolvedValue([
        {
          metadata: {
            inputMaterialization: {
              document: {
                injectedInstruction: 'stale injected instruction',
                injectionPlacement: 'body',
                inputPurpose: 'Summarize an uploaded policy draft',
                wrapperSummary: 'stale wrapper summary',
              },
            },
            pluginConfig: { inputs },
            pluginId: 'prompt-extraction',
          },
          vars: {
            [MULTI_INPUT_VAR]: baseMultiInputPrompt,
            document: 'stale-document-data-uri',
            question: 'base question',
          },
        },
      ]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'prompt-extraction',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases: any[]) => testCases);
      vi.spyOn(Strategies, 'find').mockReturnValue({
        action: mockStrategyAction,
        id: 'jailbreak:meta',
      });

      const result = await synthesize({
        inputs,
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'prompt-extraction', numTests: 1 }],
        prompts: ['{{document}} {{question}}'],
        provider: mockProvider,
        purpose: 'Summarize uploaded documents',
        strategies: [{ id: 'jailbreak:meta' }],
        targetIds: ['test-provider'],
      });

      const strategyTestCase = result.testCases.find(
        (testCase) => testCase.metadata?.strategyId === 'jailbreak:meta',
      );

      expect(strategyTestCase).toBeDefined();
      expect(mockProvider.callApi).not.toHaveBeenCalled();
      expect(strategyTestCase?.vars?.document).toBe('stale-document-data-uri');
      expect(strategyTestCase?.metadata?.inputMaterialization).toMatchObject({
        document: {
          injectedInstruction: 'stale injected instruction',
          injectionPlacement: 'body',
          inputPurpose: 'Summarize an uploaded policy draft',
          wrapperSummary: 'stale wrapper summary',
        },
      });
    });

    it('should find exact strategy ID for custom strategy', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockCustomAction = vi.fn().mockReturnValue([
        {
          vars: { query: 'custom test' },
          metadata: { strategyId: 'custom' },
        },
      ]);

      // Mock the Strategies array to include the exact 'custom' strategy
      vi.spyOn(Strategies, 'find').mockImplementation(function (predicate) {
        if (typeof predicate === 'function') {
          const strategies = [{ id: 'custom', action: mockCustomAction }];
          return strategies.find(predicate);
        }
        return undefined;
      });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'custom' }],
        targetIds: ['test-provider'],
      });

      // Should have found the exact 'custom' strategy and called its action
      expect(mockCustomAction).toHaveBeenCalled();

      // Check that the strategy test cases have the correct strategy ID
      const strategyTestCases = result.testCases.filter((tc) => tc.metadata?.strategyId);
      expect(strategyTestCases).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            metadata: expect.objectContaining({ strategyId: 'custom' }),
          }),
        ]),
      );
    });

    it('should fall back to base strategy ID for custom variants', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockCustomAction = vi.fn().mockReturnValue([
        {
          vars: { query: 'custom test' },
          metadata: { strategyId: 'custom:aggressive' },
        },
      ]);

      // Mock the Strategies array to include only the base 'custom' strategy
      vi.spyOn(Strategies, 'find').mockImplementation(function (predicate) {
        if (typeof predicate === 'function') {
          const strategies = [{ id: 'custom', action: mockCustomAction }];
          return strategies.find(predicate);
        }
        return undefined;
      });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'custom:aggressive' }],
        targetIds: ['test-provider'],
      });

      // Should have found the base 'custom' strategy and called its action
      expect(mockCustomAction).toHaveBeenCalled();

      // Check that the strategy test cases have the correct strategy ID
      const strategyTestCases = result.testCases.filter((tc) => tc.metadata?.strategyId);
      expect(strategyTestCases).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            metadata: expect.objectContaining({ strategyId: 'custom:aggressive' }),
          }),
        ]),
      );
    });

    it('should warn when strategy is not found', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      // Mock Strategies.find to return undefined (strategy not found)
      vi.spyOn(Strategies, 'find').mockReturnValue(undefined);

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'nonexistent-strategy' }],
        targetIds: ['test-provider'],
      });

      expect(logger.warn).toHaveBeenCalledWith(
        'Strategy nonexistent-strategy not registered, skipping',
      );
    });

    it('should prioritize exact strategy match over base strategy for colon-separated IDs', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      const mockJailbreakAction = vi.fn().mockReturnValue([
        {
          vars: { query: 'basic jailbreak' },
          metadata: { strategyId: 'jailbreak' },
        },
      ]);

      const mockJailbreakCompositeAction = vi.fn().mockReturnValue([
        {
          vars: { query: 'composite jailbreak' },
          metadata: { strategyId: 'jailbreak:composite' },
        },
      ]);

      // Mock the Strategies array to include both strategies
      vi.spyOn(Strategies, 'find').mockImplementation(function (predicate) {
        if (typeof predicate === 'function') {
          const strategies = [
            { id: 'jailbreak', action: mockJailbreakAction },
            { id: 'jailbreak:composite', action: mockJailbreakCompositeAction },
          ];
          return strategies.find(predicate);
        }
        return undefined;
      });

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak:composite' }],
        targetIds: ['test-provider'],
      });

      // Should have called the composite action, not the basic jailbreak action
      expect(mockJailbreakCompositeAction).toHaveBeenCalled();
      expect(mockJailbreakAction).not.toHaveBeenCalled();
    });

    it('should skip plugins that fail validation and not throw', async () => {
      const failingPlugin = {
        id: 'fail-plugin',
        numTests: 1,
      };
      const passingPlugin = {
        id: 'pass-plugin',
        numTests: 1,
      };

      vi.spyOn(Plugins, 'find')
        .mockReturnValueOnce({
          key: 'fail-plugin',
          action: vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]),
          validate: () => {
            throw new Error('Validation failed!');
          },
        })
        .mockReturnValue({
          key: 'pass-plugin',
          action: vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]),
          validate: vi.fn(),
        });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [failingPlugin, passingPlugin],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(result.testCases).toHaveLength(1);
      expect(result.testCases[0].metadata.pluginId).toBe('pass-plugin');
      expect(logger.warn).toHaveBeenCalledWith(
        expect.stringContaining(
          'Validation failed for plugin fail-plugin: Error: Validation failed!, skipping plugin',
        ),
      );
    });

    it('should not store full config in metadata for intent plugin to prevent bloating', async () => {
      vi.clearAllMocks();

      const mockProvider = {
        id: () => 'test-provider',
        callApi: vi.fn().mockResolvedValue({ output: 'Test response' }),
      };

      const intentPlugin = {
        id: 'intent',
        numTests: 2,
        config: {
          intent: ['intent1', 'intent2', 'intent3', 'intent4', 'intent5'],
        },
      };

      const regularPlugin = {
        id: 'contracts',
        numTests: 1,
        config: {
          someConfig: 'value',
        },
      };

      const mockIntentAction = vi.fn().mockResolvedValue([
        {
          vars: { prompt: 'intent1' },
          assert: [{ type: 'promptfoo:redteam:intent', metric: 'Intent' }],
          metadata: {
            intent: 'intent1',
            pluginId: 'intent',
            pluginConfig: undefined,
          },
        },
        {
          vars: { prompt: 'intent2' },
          assert: [{ type: 'promptfoo:redteam:intent', metric: 'Intent' }],
          metadata: {
            intent: 'intent2',
            pluginId: 'intent',
            pluginConfig: undefined,
          },
        },
      ]);

      const mockContractsAction = vi.fn().mockResolvedValue([
        {
          vars: { prompt: 'contract test' },
          assert: [{ type: 'promptfoo:redteam:contracts', metric: 'Contracts' }],
          metadata: {
            pluginId: 'contracts',
          },
        },
      ]);

      vi.spyOn(Plugins, 'find').mockImplementation(function (predicate) {
        const mockPlugins = [
          { key: 'intent', action: mockIntentAction },
          { key: 'contracts', action: mockContractsAction },
        ];

        if (typeof predicate === 'function') {
          return mockPlugins.find(predicate);
        }
        return undefined;
      });

      const result = await synthesize({
        plugins: [intentPlugin, regularPlugin],
        prompts: ['Test prompt'],
        provider: mockProvider,
        purpose: 'Test purpose',
        strategies: [],
        injectVar: 'prompt',
        language: 'en',
        numTests: 5,
        targetIds: ['test'],
      });

      const intentTestCases = result.testCases.filter((tc) => tc.metadata?.pluginId === 'intent');
      expect(intentTestCases.length).toBeGreaterThan(0);
      intentTestCases.forEach((tc) => {
        expect(tc.metadata?.pluginConfig).toBeUndefined();
        expect(tc.metadata?.pluginId).toBe('intent');
      });

      const contractsTestCases = result.testCases.filter(
        (tc) => tc.metadata?.pluginId === 'contracts',
      );
      expect(contractsTestCases.length).toBeGreaterThan(0);
      contractsTestCases.forEach((tc) => {
        expect(tc.metadata?.pluginConfig).toBeDefined();
        expect(tc.metadata?.pluginConfig).toEqual({ someConfig: 'value' });
        expect(tc.metadata?.pluginId).toBe('contracts');
      });
    });

    it('should handle strategies that return undefined test cases', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([
        { vars: { query: 'test1' }, metadata: { pluginId: 'test-plugin' } },
        { vars: { query: 'test2' }, metadata: { pluginId: 'test-plugin' } },
      ]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      // Mock a strategy that returns an array with some undefined values
      const mockStrategyAction = vi.fn().mockReturnValue([
        { vars: { query: 'strategy test 1' }, metadata: { strategyId: 'test-strategy' } },
        undefined, // This undefined should be filtered out
        { vars: { query: 'strategy test 2' }, metadata: { strategyId: 'test-strategy' } },
        undefined, // This undefined should be filtered out
      ]);

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'test-strategy',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        language: 'en',
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'test-strategy' }],
        targetIds: ['test-provider'],
      });

      // Should have 2 base tests + 2 strategy tests (undefined values filtered out)
      expect(result.testCases).toHaveLength(4);

      // Verify that no undefined values made it through
      expect(result.testCases.every((tc) => tc !== undefined && tc !== null)).toBe(true);

      // Verify strategy tests are present (only the non-undefined ones)
      const strategyTests = result.testCases.filter((tc) => tc.metadata?.strategyId);
      expect(strategyTests).toHaveLength(2);
    });

    it('should handle strategies that return all undefined test cases', async () => {
      const mockPluginAction = vi
        .fn()
        .mockResolvedValue([{ vars: { query: 'test1' }, metadata: { pluginId: 'test-plugin' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      // Mock a strategy that returns only undefined values
      const mockStrategyAction = vi.fn().mockReturnValue([undefined, undefined, undefined]);

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'failing-strategy',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'failing-strategy' }],
        targetIds: ['test-provider'],
      });

      // Should only have the base test (no strategy tests since all were undefined)
      expect(result.testCases).toHaveLength(1);
      expect(result.testCases[0].metadata?.pluginId).toBe('test-plugin');
      expect(result.testCases[0].metadata?.strategyId).toBeUndefined();
    });

    it('should preserve language metadata when filtering undefined strategy test cases', async () => {
      const mockPluginAction = vi
        .fn()
        .mockResolvedValue([
          { vars: { query: 'test1' }, metadata: { pluginId: 'test-plugin', language: 'fr' } },
        ]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ action: mockPluginAction, key: 'mockPlugin' });

      // Mock a strategy that returns mixed undefined and valid test cases with language metadata
      const mockStrategyAction = vi.fn().mockReturnValue([
        {
          vars: { query: 'strategy test 1' },
          metadata: { strategyId: 'test-strategy', language: 'fr' },
        },
        undefined,
        {
          vars: { query: 'strategy test 2' },
          metadata: { strategyId: 'test-strategy', language: 'fr' },
        },
      ]);

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'test-strategy',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        language: 'fr',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'test-strategy' }],
        targetIds: ['test-provider'],
      });

      // Should have 1 base test + 2 strategy tests
      expect(result.testCases).toHaveLength(3);

      // All test cases should have language metadata preserved
      result.testCases.forEach((tc) => {
        expect(tc.metadata?.language).toBe('fr');
      });

      // Verify strategy tests preserved language
      const strategyTests = result.testCases.filter((tc) => tc.metadata?.strategyId);
      expect(strategyTests).toHaveLength(2);
      strategyTests.forEach((tc) => {
        expect(tc.metadata?.language).toBe('fr');
      });
    });
  });

  describe('Logger', () => {
    it('debug log level hides progress bar', async () => {
      const restoreEnv = mockProcessEnv({ LOG_LEVEL: 'debug' });
      try {
        await synthesize({
          language: 'en',
          numTests: 1,
          plugins: [{ id: 'test-plugin', numTests: 1 }],
          prompts: ['Test prompt'],
          strategies: [],
          targetIds: ['test-provider'],
        });

        expect(cliProgress.SingleBar).not.toHaveBeenCalled();
      } finally {
        restoreEnv();
      }
    });
  });

  describe('Progress bar', () => {
    it('should initialize progress bar with totalTests including strategies', async () => {
      const mockStart = vi.fn();
      const mockIncrement = vi.fn();
      const mockUpdate = vi.fn();
      const mockStop = vi.fn();

      vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
        return {
          start: mockStart,
          increment: mockIncrement,
          update: mockUpdate,
          stop: mockStop,
        } as any;
      });

      // Mock plugin to return test cases
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      // 2 plugins × 1 test = 2 plugin tests
      // 2 strategies × 2 tests = 4 strategy tests (jailbreak strategies add 1:1)
      // Total = 6 tests
      await synthesize({
        numTests: 1,
        plugins: [
          { id: 'test-plugin', numTests: 1 },
          { id: 'test-plugin', numTests: 1 },
        ],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak' }, { id: 'jailbreak:tree' }],
        targetIds: ['test-provider'],
      });

      // Progress bar should be started with totalTests (not just plugin tests)
      expect(mockStart).toHaveBeenCalled();
      const startCall = mockStart.mock.calls[0];
      // totalTests should be greater than just plugin tests (2)
      expect(startCall[0]).toBeGreaterThan(2);
    });

    it('should increment progress bar for strategy tests', async () => {
      const mockStart = vi.fn();
      const mockIncrement = vi.fn();
      const mockUpdate = vi.fn();
      const mockStop = vi.fn();

      vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
        return {
          start: mockStart,
          increment: mockIncrement,
          update: mockUpdate,
          stop: mockStop,
        } as any;
      });

      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      await synthesize({
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak' }],
        targetIds: ['test-provider'],
      });

      // Should have increment calls for both plugin tests and strategy tests
      expect(mockIncrement).toHaveBeenCalled();
      // At least one increment call should be for strategy tests (with a number argument)
      const incrementCalls = mockIncrement.mock.calls;
      const hasStrategyIncrement = incrementCalls.some(
        (call) => typeof call[0] === 'number' && call[0] > 0,
      );
      expect(hasStrategyIncrement).toBe(true);
    });

    it('should include strategy tests in progress bar for multilingual configs', async () => {
      const mockStart = vi.fn();
      const mockIncrement = vi.fn();
      const mockUpdate = vi.fn();
      const mockStop = vi.fn();

      vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
        return {
          start: mockStart,
          increment: mockIncrement,
          update: mockUpdate,
          stop: mockStop,
        } as any;
      });

      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      // 2 plugins × 1 test × 2 languages = 4 plugin tests
      // 1 strategy × 4 plugin tests = 4 strategy tests
      // Total = 8 tests
      await synthesize({
        numTests: 1,
        language: ['Hmong', 'Zulu'],
        plugins: [
          { id: 'policy', numTests: 1, config: { policy: 'Policy 1' } },
          { id: 'policy', numTests: 1, config: { policy: 'Policy 2' } },
        ],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak' }],
        targetIds: ['test-provider'],
      });

      // Progress bar should be started with totalTests including strategies
      expect(mockStart).toHaveBeenCalled();
      const startCall = mockStart.mock.calls[0];
      // Should be 8 (4 plugin + 4 strategy), not 4 (just plugins)
      expect(startCall[0]).toBeGreaterThanOrEqual(8);
    });
  });

  describe('Report status for policy plugins', () => {
    it('should show Failed status when plugin generates 0 test cases', async () => {
      // Mock plugin to return empty array (failed generation)
      const mockPluginAction = vi.fn().mockResolvedValue([]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      // Capture the report output
      let reportMessage: string | undefined;
      vi.mocked(logger.info).mockImplementation(function (msg: any) {
        if (typeof msg === 'string' && msg.includes('Test Generation Report')) {
          reportMessage = msg;
        }
        return logger as any;
      });

      await synthesize({
        numTests: 2,
        plugins: [{ id: 'policy', numTests: 2, config: { policy: 'Test policy' } }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(reportMessage).toBeDefined();
      const cleanReport = stripAnsi(reportMessage || '');
      // Should show 2 requested, 0 generated
      expect(cleanReport).toContain('2');
      expect(cleanReport).toContain('0');
      expect(cleanReport).toContain('Failed');
    });

    it('should show Partial status when plugin generates fewer tests than requested', async () => {
      // Mock plugin to return only 1 test case when 2 are requested
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test1' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      // Capture the report output
      let reportMessage: string | undefined;
      vi.mocked(logger.info).mockImplementation(function (msg: any) {
        if (typeof msg === 'string' && msg.includes('Test Generation Report')) {
          reportMessage = msg;
        }
        return logger as any;
      });

      await synthesize({
        numTests: 2,
        plugins: [{ id: 'policy', numTests: 2, config: { policy: 'Test policy' } }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(reportMessage).toBeDefined();
      const cleanReport = stripAnsi(reportMessage || '');
      // Should show 2 requested, 1 generated
      expect(cleanReport).toContain('2');
      expect(cleanReport).toContain('1');
      expect(cleanReport).toContain('Partial');
    });

    it('should show individual status for each policy when one fails and others succeed', async () => {
      // First policy returns 0, second policy returns 2
      let callCount = 0;
      const mockPluginAction = vi.fn().mockImplementation(() => {
        callCount++;
        if (callCount === 1) {
          return Promise.resolve([]); // First policy fails
        }
        return Promise.resolve([{ vars: { query: 'test1' } }, { vars: { query: 'test2' } }]);
      });

      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      // Capture the report output
      let reportMessage: string | undefined;
      vi.mocked(logger.info).mockImplementation(function (msg: any) {
        if (typeof msg === 'string' && msg.includes('Test Generation Report')) {
          reportMessage = msg;
        }
        return logger as any;
      });

      await synthesize({
        numTests: 2,
        plugins: [
          { id: 'policy', numTests: 2, config: { policy: 'Failing policy' } },
          { id: 'policy', numTests: 2, config: { policy: 'Succeeding policy' } },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(reportMessage).toBeDefined();
      const cleanReport = stripAnsi(reportMessage || '');
      // Should have both policies with display format "policy [hash]: preview..."
      expect(cleanReport).toMatch(/policy \[[a-f0-9]{12}\]:/);
      // Inline policies show hash and preview
      // Should show both Failed and Success statuses
      expect(cleanReport).toContain('Failed');
      expect(cleanReport).toContain('Success');
    });
  });

  describe('API Health Check', () => {
    beforeEach(() => {
      vi.clearAllMocks();
      vi.resetAllMocks();

      // Reset logger mocks
      vi.mocked(logger.info).mockImplementation(function () {
        return logger as any;
      });
      vi.mocked(logger.warn).mockImplementation(function () {
        return logger as any;
      });
      vi.mocked(logger.error).mockImplementation(function () {
        return logger as any;
      });
      vi.mocked(logger.debug).mockImplementation(function () {
        return logger as any;
      });

      // Set up templates mock with consistent default behavior
      vi.mocked(extractVariablesFromTemplates).mockImplementation(function () {
        return ['query'];
      });

      vi.mocked(shouldGenerateRemote).mockImplementation(function () {
        return true;
      });
      vi.mocked(getRemoteHealthUrl).mockImplementation(function () {
        return 'https://api.test/health';
      });
      vi.mocked(checkRemoteHealth).mockResolvedValue({
        status: 'OK',
        message: 'Cloud API is healthy',
      });
    });

    it('should check API health when remote generation is enabled', async () => {
      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(shouldGenerateRemote).toHaveBeenCalledWith();
      expect(getRemoteHealthUrl).toHaveBeenCalledWith();
      expect(checkRemoteHealth).toHaveBeenCalledWith('https://api.test/health');
    });

    it('should skip health check when remote generation is disabled', async () => {
      vi.mocked(shouldGenerateRemote).mockImplementation(function () {
        return false;
      });

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(shouldGenerateRemote).toHaveBeenCalledWith();
      expect(getRemoteHealthUrl).not.toHaveBeenCalled();
      expect(checkRemoteHealth).not.toHaveBeenCalled();
    });

    it('should throw error when health check fails', async () => {
      vi.mocked(checkRemoteHealth).mockResolvedValue({
        status: 'ERROR',
        message: 'API is not accessible',
      });

      await expect(
        synthesize({
          language: 'en',
          numTests: 1,
          plugins: [{ id: 'test-plugin', numTests: 1 }],
          prompts: ['Test prompt'],
          strategies: [],
          targetIds: ['test-provider'],
        }),
      ).rejects.toThrow('Unable to proceed with test generation: API is not accessible');
    });

    it('should skip health check when URL is null', async () => {
      vi.mocked(getRemoteHealthUrl).mockImplementation(function () {
        return null;
      });

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(shouldGenerateRemote).toHaveBeenCalledWith();
      expect(getRemoteHealthUrl).toHaveBeenCalledWith();
      expect(checkRemoteHealth).not.toHaveBeenCalled();
    });
  });

  it('should handle basic strategy configuration', async () => {
    vi.mocked(loadApiProvider).mockResolvedValue({
      id: () => 'test',
      callApi: vi.fn().mockResolvedValue({ output: 'test output' }),
    });

    const mockPlugin = {
      id: 'test-plugin',
      numTests: 1,
    };

    const mockProvider = {
      id: () => 'test',
      callApi: vi.fn().mockResolvedValue({ output: 'test output' }),
    };

    const mockTestPluginAction = vi.fn().mockResolvedValue([
      {
        vars: { input: 'test input' },
        assert: [{ type: 'test-assertion', metric: 'Test' }],
        metadata: {
          pluginId: 'test-plugin',
        },
      },
    ]);

    vi.spyOn(Plugins, 'find').mockReturnValue({
      key: 'test-plugin',
      action: mockTestPluginAction,
    });

    const resultEnabled = await synthesize({
      plugins: [mockPlugin],
      strategies: [{ id: 'basic', config: { enabled: true } }],
      prompts: ['test prompt'],
      injectVar: 'input',
      provider: mockProvider,
      language: 'en',
      numTests: 1,
      targetIds: ['test-provider'],
    });

    expect(resultEnabled.testCases.length).toBeGreaterThan(0);

    const resultDisabled = await synthesize({
      plugins: [mockPlugin],
      strategies: [{ id: 'basic', config: { enabled: false } }],
      prompts: ['test prompt'],
      injectVar: 'input',
      provider: mockProvider,
      language: 'en',
      numTests: 1,
      targetIds: ['test-provider'],
    });

    expect(resultDisabled.testCases).toHaveLength(0);
  });

  describe('Direct plugin handling', () => {
    it('should recognize and not expand direct plugins like bias:gender', async () => {
      const mockPluginAction = vi.fn().mockImplementation(function ({ n }) {
        return Array(n).fill({ vars: { query: 'test' } });
      });
      vi.spyOn(Plugins, 'find').mockReturnValue({ key: 'bias:gender', action: mockPluginAction });

      const result = await synthesize({
        language: 'en',
        numTests: 2,
        plugins: [{ id: 'bias:gender', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Check that the plugin wasn't expanded and was used directly
      expect(mockPluginAction).toHaveBeenCalledWith(
        expect.objectContaining({
          provider: expect.anything(),
          purpose: expect.any(String),
          n: 2,
        }),
      );

      // Check that the test cases have the correct plugin ID
      const testCases = result.testCases;
      testCases.forEach((tc) => {
        expect(tc.metadata.pluginId).toBe('bias:gender');
      });

      // Should have exactly the number of test cases we requested
      expect(testCases).toHaveLength(2);
    });

    it('should still expand category plugins with new bias category', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({ key: 'mockPlugin', action: mockPluginAction });

      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'bias', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Check that we have test cases for each bias plugin
      const biasPluginIds = Object.keys(HARM_PLUGINS).filter((p) => p.startsWith('bias:'));
      const testCasePluginIds = result.testCases.map((tc) => tc.metadata.pluginId);

      // Every bias plugin should have a test case
      biasPluginIds.forEach((id) => {
        expect(testCasePluginIds).toContain(id);
      });
    });
  });
});

vi.mock('fs');
vi.mock('../../src/util/yamlLoad');

describe('resolvePluginConfig', () => {
  beforeEach(() => {
    vi.resetAllMocks();

    // Set up logger mocks
    vi.mocked(logger.info).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.warn).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.error).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.debug).mockImplementation(function () {
      return logger as any;
    });
  });

  afterEach(() => {
    vi.clearAllMocks();
  });

  it('should return an empty object if config is undefined', () => {
    const result = resolvePluginConfig(undefined);
    expect(result).toEqual({});
  });

  it('should return the original config if no file references are present', () => {
    const config = { key: 'value' };
    const result = resolvePluginConfig(config);
    expect(result).toEqual(config);
  });

  it('should resolve YAML file references', () => {
    const config = { key: 'file://test.yaml' };
    const yamlContent = { nested: 'value' };
    vi.spyOn(fs, 'existsSync').mockReturnValue(true);
    vi.spyOn(fs, 'readFileSync').mockReturnValue('yaml content');
    vi.mocked(loadYaml).mockImplementation(function () {
      return yamlContent;
    });

    const result = resolvePluginConfig(config);

    expect(result).toEqual({ key: yamlContent });
    expect(fs.existsSync).toHaveBeenCalledWith('test.yaml');
    expect(fs.readFileSync).toHaveBeenCalledWith('test.yaml', 'utf8');
    expect(loadYaml).toHaveBeenCalledWith('yaml content');
  });

  it('should resolve JSON file references', () => {
    const config = { key: 'file://test.json' };
    const jsonContent = { nested: 'value' };
    vi.spyOn(fs, 'existsSync').mockReturnValue(true);
    vi.spyOn(fs, 'readFileSync').mockReturnValue(JSON.stringify(jsonContent));

    const result = resolvePluginConfig(config);

    expect(result).toEqual({ key: jsonContent });
    expect(fs.existsSync).toHaveBeenCalledWith('test.json');
    expect(fs.readFileSync).toHaveBeenCalledWith('test.json', 'utf8');
  });

  it('should resolve text file references', () => {
    const config = { key: 'file://test.txt' };
    const fileContent = 'text content';
    vi.spyOn(fs, 'existsSync').mockReturnValue(true);
    vi.spyOn(fs, 'readFileSync').mockReturnValue(fileContent);

    const result = resolvePluginConfig(config);

    expect(result).toEqual({ key: fileContent });
    expect(fs.existsSync).toHaveBeenCalledWith('test.txt');
    expect(fs.readFileSync).toHaveBeenCalledWith('test.txt', 'utf8');
  });

  it('should throw an error if the file does not exist', () => {
    const config = { key: 'file://nonexistent.yaml' };
    vi.spyOn(fs, 'existsSync').mockReturnValue(false);

    expect(() => resolvePluginConfig(config)).toThrow('File not found: nonexistent.yaml');
  });

  it('should handle multiple file references', () => {
    const config = {
      yaml: 'file://test.yaml',
      json: 'file://test.json',
      txt: 'file://test.txt',
    };
    const yamlContent = { nested: 'yaml' };
    const jsonContent = { nested: 'json' };
    const txtContent = 'text content';

    vi.spyOn(fs, 'existsSync').mockReturnValue(true);
    vi.spyOn(fs, 'readFileSync')
      .mockReturnValueOnce('yaml content')
      .mockReturnValueOnce(JSON.stringify(jsonContent))
      .mockReturnValueOnce(txtContent);
    vi.mocked(loadYaml).mockImplementation(function () {
      return yamlContent;
    });

    const result = resolvePluginConfig(config);

    expect(result).toEqual({
      yaml: yamlContent,
      json: jsonContent,
      txt: txtContent,
    });
  });
});

describe('calculateTotalTests', () => {
  const mockPlugins = [
    { id: 'plugin1', numTests: 2 },
    { id: 'plugin2', numTests: 3 },
  ];

  it('should calculate basic test counts with no strategies', () => {
    const result = calculateTotalTests(mockPlugins, []);
    expect(result).toEqual({
      totalTests: 5,
      totalPluginTests: 5,
      effectiveStrategyCount: 0,
      includeBasicTests: true,
    });
  });

  it('should handle basic strategy when enabled', () => {
    const strategies = [{ id: 'basic', config: { enabled: true } }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 5,
      totalPluginTests: 5,
      effectiveStrategyCount: 1,
      includeBasicTests: true,
    });
  });

  it('should handle basic strategy when disabled', () => {
    const strategies = [{ id: 'basic', config: { enabled: false } }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 0,
      totalPluginTests: 5,
      effectiveStrategyCount: 0,
      includeBasicTests: false,
    });
  });

  it('should count custom intents instead of numTests for intent plugins', () => {
    const plugins = [
      {
        id: 'intent',
        numTests: 1,
        config: { intent: ['intent1', 'intent2', 'intent3'] },
      },
    ];

    const result = calculateTotalTests(plugins, []);

    expect(result).toEqual({
      totalTests: 3,
      totalPluginTests: 3,
      effectiveStrategyCount: 0,
      includeBasicTests: true,
    });
  });

  it('should multiply intent count by language count for multilingual intent plugins', () => {
    const plugins = [
      {
        id: 'intent',
        numTests: 1,
        config: {
          intent: ['intent1', 'intent2'],
          language: ['en', 'es', 'fr'],
        },
      },
    ];

    const result = calculateTotalTests(plugins, []);

    expect(result).toEqual({
      totalTests: 6,
      totalPluginTests: 6,
      effectiveStrategyCount: 0,
      includeBasicTests: true,
    });
  });

  it('should resolve file:// intents from a JSON array for pre-generation totals', () => {
    // fs is mocked at the module level (see vi.mock('fs') below). Stub existsSync/readFileSync
    // so maybeLoadFromExternalFile sees an existing file with a JSON array of intents.
    const existsSyncSpy = vi.spyOn(fs, 'existsSync').mockReturnValue(true);
    const readFileSyncSpy = vi
      .spyOn(fs, 'readFileSync')
      .mockReturnValue(JSON.stringify(['do thing 1', 'do thing 2', 'do thing 3', 'do thing 4']));

    try {
      const plugins = [
        {
          id: 'intent',
          numTests: 1,
          config: { intent: 'file://intents.json' },
        },
      ];

      const result = calculateTotalTests(plugins, []);

      expect(result).toEqual({
        totalTests: 4,
        totalPluginTests: 4,
        effectiveStrategyCount: 0,
        includeBasicTests: true,
      });
    } finally {
      readFileSyncSpy.mockRestore();
      existsSyncSpy.mockRestore();
    }
  });

  it('should fall back to count of 1 when file:// intent path is unreadable', () => {
    const plugins = [
      {
        id: 'intent',
        numTests: 1,
        config: { intent: 'file:///definitely/does/not/exist/intents.txt' },
      },
    ];

    const result = calculateTotalTests(plugins, []);

    // The file can't be read, so the helper falls back to treating the literal
    // string as a single intent. This matches the pre-fix behavior and keeps
    // the helper from throwing during early UI calculations.
    expect(result.totalPluginTests).toBe(1);
  });

  it('should report 0 plugin tests when an intent plugin is missing config.intent', () => {
    const plugins = [
      {
        id: 'intent',
        numTests: 5,
        config: {},
      },
    ];

    const result = calculateTotalTests(plugins, []);

    expect(result.totalPluginTests).toBe(0);
  });

  it('should handle retry strategy with default numTests', () => {
    const strategies = [{ id: 'retry' }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 10,
      totalPluginTests: 5,
      effectiveStrategyCount: 1,
      includeBasicTests: true,
    });
  });

  it('should handle retry strategy with custom numTests', () => {
    const strategies = [{ id: 'retry', config: { numTests: 3 } }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 8,
      totalPluginTests: 5,
      effectiveStrategyCount: 1,
      includeBasicTests: true,
    });
  });

  it('should handle retry strategy combined with other strategies', () => {
    const strategies = [{ id: 'retry' }, { id: 'rot13' }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 15,
      totalPluginTests: 5,
      effectiveStrategyCount: 2,
      includeBasicTests: true,
    });
  });

  it('should correctly calculate total tests for multiple plugins with jailbreak strategy', () => {
    const plugins = Array(10).fill({ numTests: 5 });
    const strategies = [{ id: 'jailbreak' }];
    const result = calculateTotalTests(plugins, strategies);
    expect(result).toEqual({
      totalTests: 100,
      totalPluginTests: 50,
      effectiveStrategyCount: 1,
      includeBasicTests: true,
    });
  });

  it('should add tests for each strategy instead of replacing the total', () => {
    const strategies = [{ id: 'morse' }, { id: 'piglatin' }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 15,
      totalPluginTests: 5,
      effectiveStrategyCount: 2,
      includeBasicTests: true,
    });
  });

  it('should handle multiple strategies', () => {
    const strategies = [{ id: 'morse' }, { id: 'piglatin' }, { id: 'rot13' }];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 20,
      totalPluginTests: 5,
      effectiveStrategyCount: 3,
      includeBasicTests: true,
    });
  });

  it('should handle multiple strategies with basic strategy disabled', () => {
    const strategies = [
      { id: 'basic', config: { enabled: false } },
      { id: 'morse' },
      { id: 'piglatin' },
    ];
    const result = calculateTotalTests(mockPlugins, strategies);
    expect(result).toEqual({
      totalTests: 10,
      totalPluginTests: 5,
      effectiveStrategyCount: 2,
      includeBasicTests: false,
    });
  });
});

describe('getTestCount', () => {
  it('should return totalPluginTests when basic strategy is enabled', () => {
    const strategy = { id: 'basic', config: { enabled: true } };
    const result = getTestCount(strategy, 10, []);
    expect(result).toBe(10);
  });

  it('should return 0 when basic strategy is disabled', () => {
    const strategy = { id: 'basic', config: { enabled: false } };
    const result = getTestCount(strategy, 10, []);
    expect(result).toBe(0);
  });

  it('should add configured number of tests for retry strategy', () => {
    const strategy = { id: 'retry', config: { numTests: 5 } };
    const result = getTestCount(strategy, 10, []);
    expect(result).toBe(15);
  });

  it('should add totalPluginTests for retry strategy when numTests not specified', () => {
    const strategy = { id: 'retry' };
    const result = getTestCount(strategy, 10, []);
    expect(result).toBe(20);
  });

  it('should return totalPluginTests for other strategies', () => {
    const strategy = { id: 'morse' };
    const result = getTestCount(strategy, 10, []);
    expect(result).toBe(10);
  });

  it('should return totalPluginTests for layer strategy', () => {
    const strategy = {
      id: 'layer',
      config: {
        steps: ['base64', 'rot13'],
      },
    };
    const result = getTestCount(strategy, 10, []);
    expect(result).toBe(10);
  });

  describe('numTests cap', () => {
    it('should cap test count when numTests is less than calculated count', () => {
      // Fan-out strategy with default n=5 would produce 50 tests
      const strategy = { id: 'jailbreak', config: { numTests: 10 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(10);
    });

    it('should return 0 when numTests is 0', () => {
      const strategy = { id: 'base64', config: { numTests: 0 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(0);
    });

    it('should not affect count when numTests is larger than calculated', () => {
      // Non-fanout strategy produces 10 tests (1:1)
      const strategy = { id: 'morse', config: { numTests: 100 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(10);
    });

    it('should cap basic strategy tests', () => {
      const strategy = { id: 'basic', config: { numTests: 5 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(5);
    });

    it('should cap layer strategy tests', () => {
      const strategy = {
        id: 'layer',
        config: {
          numTests: 3,
          steps: ['base64', 'rot13'],
        },
      };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(3);
    });

    it('should not apply numTests cap to retry strategy (different semantics)', () => {
      // Retry has additive semantics: totalPluginTests + numTests
      const strategy = { id: 'retry', config: { numTests: 5 } };
      const result = getTestCount(strategy, 10, []);
      // Should be 10 + 5 = 15, not capped
      expect(result).toBe(15);
    });

    it('should cap fan-out strategy with custom n and numTests', () => {
      // n=3 would produce 30 tests, but numTests caps at 15
      const strategy = { id: 'jailbreak', config: { numTests: 15, n: 3 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(15);
    });

    it('should handle numTests equal to calculated count (no-op)', () => {
      // Non-fanout strategy produces 10 tests (1:1), numTests also set to 10
      const strategy = { id: 'morse', config: { numTests: 10 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(10);
    });

    it('should handle numTests equal to fan-out calculated count (no-op)', () => {
      // Fan-out with explicit n=5 produces 50 tests, numTests also set to 50
      const strategy = { id: 'jailbreak', config: { numTests: 50, n: 5 } };
      const result = getTestCount(strategy, 10, []);
      expect(result).toBe(50);
    });

    it('should ignore invalid numTests values at runtime (NaN)', () => {
      // Defensive check should skip NaN
      const strategy = { id: 'base64', config: { numTests: NaN } };
      const result = getTestCount(strategy, 10, []);
      // Should return uncapped count (10 for non-fanout)
      expect(result).toBe(10);
    });

    it('should ignore invalid numTests values at runtime (Infinity)', () => {
      // Defensive check should skip Infinity
      const strategy = { id: 'base64', config: { numTests: Infinity } };
      const result = getTestCount(strategy, 10, []);
      // Should return uncapped count (10 for non-fanout)
      expect(result).toBe(10);
    });

    it('should ignore invalid numTests values at runtime (negative)', () => {
      // Defensive check should skip negative values
      const strategy = { id: 'base64', config: { numTests: -5 } };
      const result = getTestCount(strategy, 10, []);
      // Should return uncapped count (10 for non-fanout)
      expect(result).toBe(10);
    });
  });
});

describe('Language configuration', () => {
  const mockProvider = {
    callApi: vi.fn(),
    generate: vi.fn(),
    id: () => 'test-provider',
  };

  beforeEach(() => {
    vi.clearAllMocks();
    vi.resetAllMocks();

    // Set up logger mocks
    vi.mocked(logger.info).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.warn).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.error).mockImplementation(function () {
      return logger as any;
    });
    vi.mocked(logger.debug).mockImplementation(function () {
      return logger as any;
    });

    // Set up templates mock
    vi.mocked(extractVariablesFromTemplates).mockImplementation(function () {
      return ['query'];
    });

    vi.mocked(extractEntities).mockResolvedValue(['entity1', 'entity2']);
    vi.mocked(extractSystemPurpose).mockResolvedValue('Test purpose');
    vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
    vi.mocked(validateStrategies).mockImplementation(async function () {});
    vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
      return {
        increment: vi.fn(),
        start: vi.fn(),
        stop: vi.fn(),
        update: vi.fn(),
      } as any;
    });
    vi.mocked(shouldGenerateRemote).mockImplementation(function () {
      return false;
    });
    vi.mocked(getRemoteHealthUrl).mockImplementation(function () {
      return 'https://api.test/health';
    });
    vi.mocked(checkRemoteHealth).mockResolvedValue({
      status: 'OK',
      message: 'OK',
    });

    // Mock plugin action to return test cases
    const mockPluginAction = vi
      .fn()
      .mockResolvedValue([{ vars: { query: 'test1' } }, { vars: { query: 'test2' } }]);
    vi.spyOn(Plugins, 'find').mockReturnValue({
      action: mockPluginAction,
      key: 'mockPlugin',
    });
  });

  describe('calculateTotalTests with language', () => {
    it('should calculate correct test count for single language', () => {
      const plugins = [
        { id: 'plugin1', numTests: 2 },
        { id: 'plugin2', numTests: 3 },
      ];
      const result = calculateTotalTests(plugins, [], 'en');
      expect(result).toEqual({
        totalTests: 5,
        totalPluginTests: 5,
        effectiveStrategyCount: 0,
        includeBasicTests: true,
      });
    });

    it('should calculate correct test count for multiple languages', () => {
      const plugins = [
        { id: 'plugin1', numTests: 2 },
        { id: 'plugin2', numTests: 3 },
      ];
      // With 3 languages, we expect 5 tests * 3 = 15 tests
      const result = calculateTotalTests(plugins, [], ['en', 'fr', 'de']);
      expect(result).toEqual({
        totalTests: 15,
        totalPluginTests: 15,
        effectiveStrategyCount: 0,
        includeBasicTests: true,
      });
    });

    it('should calculate correct test count with strategies and multiple languages', () => {
      const plugins = [
        { id: 'plugin1', numTests: 2 },
        { id: 'plugin2', numTests: 3 },
      ];
      const strategies = [{ id: 'rot13' }];
      // Base tests: 5 * 3 languages = 15
      // Strategy tests: 5 * 3 languages = 15
      // Total: 30
      const result = calculateTotalTests(plugins, strategies, ['en', 'fr', 'de']);
      expect(result).toEqual({
        totalTests: 30,
        totalPluginTests: 15,
        effectiveStrategyCount: 1,
        includeBasicTests: true,
      });
    });

    it('should handle layer strategy with multiple languages', () => {
      const plugins = [{ id: 'plugin1', numTests: 5 }];
      const strategies = [
        {
          id: 'layer',
          config: {
            steps: ['base64', 'rot13'],
          },
        },
      ];
      // With 2 languages, plugin tests: 5 * 2 = 10
      // Layer strategy adds 10 more tests
      const result = calculateTotalTests(plugins, strategies, ['en', 'fr']);
      expect(result).toEqual({
        totalTests: 20,
        totalPluginTests: 10,
        effectiveStrategyCount: 1,
        includeBasicTests: true,
      });
    });

    it('should handle retry strategy with multiple languages', () => {
      const plugins = [{ id: 'plugin1', numTests: 2 }];
      const strategies = [{ id: 'retry', config: { numTests: 2 } }];
      // With 3 languages: 2 * 3 = 6 base tests
      // Retry adds 2 tests per base test: 6 + 2 = 8
      const result = calculateTotalTests(plugins, strategies, ['en', 'fr', 'de']);
      expect(result).toEqual({
        totalTests: 8,
        totalPluginTests: 6,
        effectiveStrategyCount: 1,
        includeBasicTests: true,
      });
    });
  });

  describe('synthesize with language configuration', () => {
    it('should generate test cases with single language', async () => {
      const result = await synthesize({
        language: 'en',
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(result.testCases).toHaveLength(2);
      expect(result.testCases.every((tc) => tc.metadata?.language === 'en')).toBe(true);
    });

    it('should generate test cases for each language when multiple languages specified', async () => {
      const result = await synthesize({
        language: ['en', 'fr'],
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Should generate 2 tests * 2 languages = 4 test cases
      expect(result.testCases).toHaveLength(4);

      // Check that we have tests for both languages
      const languageCounts = result.testCases.reduce<Record<string, number>>((acc, tc) => {
        const lang = tc.metadata?.language || 'en';
        acc[lang] = (acc[lang] || 0) + 1;
        return acc;
      }, {});

      expect(languageCounts).toEqual({
        en: 2,
        fr: 2,
      });
    });

    it('should apply strategies to multilingual test cases', async () => {
      // Mock strategy action
      const mockStrategyAction = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `transformed: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'rot13' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'rot13',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        language: ['en', 'fr'],
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'rot13' }],
        targetIds: ['test-provider'],
      });

      // Base tests: 2 * 2 languages = 4
      // Strategy tests: 2 * 2 languages = 4
      // Total: 8
      expect(result.testCases.length).toBeGreaterThanOrEqual(4);

      // Verify strategy was applied to tests in both languages
      const strategyTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'rot13');
      expect(strategyTests.length).toBeGreaterThan(0);
    });

    it('should include language in test metadata', async () => {
      // Mock plugin to return only 1 test case instead of 2
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test1' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'mockPlugin',
      });

      const result = await synthesize({
        language: ['en', 'es', 'zh'],
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(result.testCases).toHaveLength(3);

      // Each test case should have language metadata
      for (const testCase of result.testCases) {
        expect(testCase.metadata?.language).toBeDefined();
        expect(['en', 'es', 'zh']).toContain(testCase.metadata?.language);
      }
    });

    it('should handle plugin-level language configuration', async () => {
      const result = await synthesize({
        language: 'en', // Global default
        numTests: 2,
        plugins: [
          {
            id: 'test-plugin',
            numTests: 2,
            config: {
              language: ['fr', 'de'], // Plugin-specific override
            },
          },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Should generate 2 tests * 2 plugin languages = 4 test cases
      expect(result.testCases).toHaveLength(4);

      // All tests should be in plugin-specified languages
      const languages = result.testCases.map((tc) => tc.metadata?.language);
      expect(languages.every((lang) => ['fr', 'de'].includes(lang || ''))).toBe(true);
    });

    it('should handle when no language is specified', async () => {
      const result = await synthesize({
        // No language specified - will use undefined which plugins interpret as 'en'
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(result.testCases).toHaveLength(2);
      // Check that all tests have the same language
      const languages = result.testCases.map((tc) => tc.metadata?.language);
      const uniqueLanguages = [...new Set(languages)];
      expect(uniqueLanguages.length).toBe(1); // Only one language used
    });

    it('should not include language in config when no language is specified', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      await synthesize({
        // No language specified
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Verify action was called with config that does NOT contain language property
      expect(mockPluginAction).toHaveBeenCalledWith(
        expect.objectContaining({
          config: expect.not.objectContaining({
            language: expect.anything(),
          }),
        }),
      );
    });

    it('should not include language in modifiers when no language is specified', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      await synthesize({
        // No language specified
        numTests: 1,
        plugins: [
          {
            id: 'test-plugin',
            numTests: 1,
            config: {
              modifiers: {
                tone: 'aggressive',
              },
            },
          },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Verify modifiers don't contain language
      expect(mockPluginAction).toHaveBeenCalledWith(
        expect.objectContaining({
          config: expect.objectContaining({
            modifiers: expect.objectContaining({
              tone: 'aggressive',
            }),
          }),
        }),
      );

      // Get the actual config that was passed
      const actualConfig = mockPluginAction.mock.calls[0][0].config;

      // Verify language is not in the config at all
      expect(actualConfig).not.toHaveProperty('language');

      // Verify modifiers don't have language key
      expect(actualConfig.modifiers).not.toHaveProperty('language');
    });

    it('should pass testGenerationInstructions through modifiers to plugin action', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          {
            id: 'test-plugin',
            numTests: 1,
            config: {
              modifiers: {
                tone: 'aggressive',
              },
            },
          },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
        testGenerationInstructions: 'Focus on edge cases',
      });

      // Verify action was called with correct config containing merged modifiers
      expect(mockPluginAction).toHaveBeenCalledWith(
        expect.objectContaining({
          config: expect.objectContaining({
            modifiers: expect.objectContaining({
              testGenerationInstructions: 'Focus on edge cases',
              tone: 'aggressive',
            }),
          }),
        }),
      );
    });
  });

  describe('Multilingual support for media strategies', () => {
    it('should support multilingual test cases for audio strategy', async () => {
      // Mock strategy action for audio
      const mockAudioAction = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `audio: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'audio' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockImplementation(function (predicate) {
        if (typeof predicate === 'function') {
          const strategies = [{ id: 'audio', action: mockAudioAction }];
          return strategies.find(predicate);
        }
        return undefined;
      });

      const result = await synthesize({
        language: ['en', 'fr', 'de'], // 3 languages
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'audio' }],
        targetIds: ['test-provider'],
      });

      // Audio strategy now supports multiple languages
      // Mock plugin always returns 2 tests (ignores numTests)
      // Base tests: 2 tests * 3 languages = 6
      // Audio strategy tests: 6 tests (applies to all base tests)
      // Total: 12 tests
      expect(result.testCases.length).toBe(12);

      // Check that audio strategy was applied to all language variants
      const audioTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'audio');
      expect(audioTests.length).toBe(6);
    });

    it('should support multilingual test cases for video strategy', async () => {
      const mockVideoAction = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `video: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'video' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'video',
        action: mockVideoAction,
      });

      const result = await synthesize({
        language: ['en', 'es'], // 2 languages
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'video' }],
        targetIds: ['test-provider'],
      });

      // Video strategy now supports multiple languages
      // Mock plugin always returns 2 tests (ignores numTests: 1)
      // Base tests: 2 tests * 2 languages = 4
      // Strategy transforms: 4 tests → 4 video tests
      // Total: 4 base + 4 video = 8 tests
      const videoTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'video');
      expect(videoTests.length).toBe(4);
      expect(result.testCases.length).toBe(8);
    });

    it('should support multilingual test cases for image strategy', async () => {
      const mockImageAction = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `image: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'image' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'image',
        action: mockImageAction,
      });

      const result = await synthesize({
        language: ['en', 'fr', 'de', 'es'], // 4 languages
        numTests: 3,
        plugins: [{ id: 'test-plugin', numTests: 3 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'image' }],
        targetIds: ['test-provider'],
      });

      // Image strategy now supports multiple languages
      // Mock plugin always returns 2 tests (ignores numTests: 3)
      // Base tests: 2 tests * 4 languages = 8
      // Strategy transforms: 8 tests → 8 image tests
      // Total: 8 base + 8 image = 16 tests
      const imageTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'image');
      expect(imageTests.length).toBe(8);
      expect(result.testCases.length).toBe(16);
    });

    it('should support multilingual test cases for jailbreak strategy', async () => {
      const mockJailbreakAction = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `jailbreak: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'jailbreak' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'jailbreak',
        action: mockJailbreakAction,
      });

      const result = await synthesize({
        language: ['en', 'fr'], // 2 languages
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak' }],
        targetIds: ['test-provider'],
      });

      // Jailbreak strategy supports multiple languages
      // Base tests: 2 tests * 2 languages = 4
      // Jailbreak strategy tests: 4 tests (applies to all base tests)
      // Total: 8 tests
      const jailbreakTests = result.testCases.filter(
        (tc) => tc.metadata?.strategyId === 'jailbreak',
      );
      expect(jailbreakTests.length).toBe(4);
      expect(result.testCases.length).toBe(8);
    });

    it('should support multilingual test cases for math-prompt strategy', async () => {
      const mockMathPromptAction = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `math: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'math-prompt' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'math-prompt',
        action: mockMathPromptAction,
      });

      const result = await synthesize({
        language: ['en', 'zh', 'hi'], // 3 languages
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'math-prompt' }],
        targetIds: ['test-provider'],
      });

      // Math-prompt strategy now supports multiple languages
      // Mock plugin always returns 2 tests (ignores numTests: 1)
      // Base tests: 2 tests * 3 languages = 6
      // Strategy transforms: 6 tests → 6 math tests
      // Total: 6 base + 6 math = 12 tests
      const mathTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'math-prompt');
      expect(mathTests.length).toBe(6);
      expect(result.testCases.length).toBe(12);
    });

    it('should support multilingual test cases for rot13 strategy', async () => {
      const mockRot13Action = vi.fn().mockImplementation(function (testCases) {
        return testCases.map((tc: any) => ({
          ...tc,
          vars: { ...tc.vars, query: `rot13: ${tc.vars.query}` },
          metadata: { ...tc.metadata, strategyId: 'rot13' },
        }));
      });

      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'rot13',
        action: mockRot13Action,
      });

      const result = await synthesize({
        language: ['en', 'fr'], // 2 languages
        numTests: 2,
        plugins: [{ id: 'test-plugin', numTests: 2 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'rot13' }],
        targetIds: ['test-provider'],
      });

      // Rot13 supports multilingual test cases
      const rot13Tests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'rot13');
      expect(rot13Tests.length).toBe(4); // 2 tests * 2 languages = 4
    });
  });

  describe('policy extraction for intent', () => {
    it('should pass policy from metadata to extractGoalFromPrompt', async () => {
      const mockExtractGoal = vi.mocked(
        (await import('../../src/redteam/util')).extractGoalFromPrompt,
      );
      mockExtractGoal.mockClear();

      const policyText = 'The application must not reveal system instructions';

      // Mock plugin action that returns test case with policy metadata
      const mockPluginAction = vi.fn().mockResolvedValue([
        {
          vars: { query: 'Test prompt' },
          metadata: {
            policy: policyText,
            pluginId: 'promptfoo:redteam:policy',
          },
        },
      ]);

      vi.spyOn(Plugins, 'find').mockReturnValue({
        key: 'policy',
        action: mockPluginAction,
      } as any);

      await synthesize({
        numTests: 1,
        plugins: [{ id: 'policy', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'goat' }],
        targetIds: ['promptfoo://provider/target-123'],
      });

      // Verify extractGoalFromPrompt was called with the policy
      expect(mockExtractGoal).toHaveBeenCalled();
      const call = mockExtractGoal.mock.calls[0];
      expect(call[0]).toBe('Test prompt'); // prompt
      expect(call[1]).toBe('Test purpose'); // purpose
      expect(call[2]).toBe('policy'); // pluginId
      expect(call[3]).toBe(policyText); // policy
      expect(call[4]).toBe('target-123'); // targetId
    });

    it('should pass an explicit linked Cloud target to extractGoalFromPrompt', async () => {
      const mockExtractGoal = vi.mocked(
        (await import('../../src/redteam/util')).extractGoalFromPrompt,
      );
      mockExtractGoal.mockClear();

      const mockPluginAction = vi.fn().mockResolvedValue([
        {
          vars: { query: 'Test prompt' },
          metadata: { pluginId: 'promptfoo:redteam:policy' },
        },
      ]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        key: 'policy',
        action: mockPluginAction,
      } as any);

      await synthesize({
        cloudTargetDatabaseId: 'linked-target-123',
        numTests: 1,
        plugins: [{ id: 'policy', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'goat' }],
        targetIds: ['file://local-provider.ts'],
      });

      expect(mockPluginAction).toHaveBeenCalledWith(
        expect.objectContaining({ targetId: 'linked-target-123' }),
      );
      expect(mockExtractGoal.mock.calls[0][4]).toBe('linked-target-123');
    });

    it('should not pass policy when metadata does not contain policy', async () => {
      const mockExtractGoal = vi.mocked(
        (await import('../../src/redteam/util')).extractGoalFromPrompt,
      );
      mockExtractGoal.mockClear();

      // Mock plugin action that returns test case WITHOUT policy metadata
      const mockPluginAction = vi.fn().mockResolvedValue([
        {
          vars: { query: 'Test prompt' },
          metadata: {
            pluginId: 'promptfoo:redteam:other',
          },
        },
      ]);

      vi.spyOn(Plugins, 'find').mockReturnValue({
        key: 'other-plugin',
        action: mockPluginAction,
      } as any);

      await synthesize({
        numTests: 1,
        plugins: [{ id: 'other-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'jailbreak:hydra' }],
        targetIds: ['test-provider'],
      });

      // Verify extractGoalFromPrompt was called WITHOUT policy (undefined)
      expect(mockExtractGoal).toHaveBeenCalled();
      const call = mockExtractGoal.mock.calls[0];
      expect(call[0]).toBe('Test prompt'); // prompt
      expect(call[1]).toBe('Test purpose'); // purpose
      expect(call[2]).toBe('other-plugin'); // pluginId
      expect(call[3]).toBeUndefined(); // policy should be undefined
    });

    it('should handle policy in metadata with safe type checking', async () => {
      const mockExtractGoal = vi.mocked(
        (await import('../../src/redteam/util')).extractGoalFromPrompt,
      );
      mockExtractGoal.mockClear();

      const testCases = [
        {
          metadata: { policy: 'Valid policy' },
          expectedPolicy: 'Valid policy',
        },
        {
          metadata: { policy: { id: 'abc123', text: 'Policy from object' } },
          expectedPolicy: 'Policy from object',
        },
        {
          metadata: { policy: '' },
          expectedPolicy: '', // Empty string is extracted but filtered later by util.ts
        },
        {
          metadata: { someOtherField: 'value' },
          expectedPolicy: undefined, // No policy field
        },
        {
          metadata: null,
          expectedPolicy: undefined, // Null metadata
        },
      ];

      for (const testCase of testCases) {
        mockExtractGoal.mockClear();

        const mockPluginAction = vi.fn().mockResolvedValue([
          {
            vars: { query: 'Test prompt' },
            metadata: testCase.metadata,
          },
        ]);

        vi.spyOn(Plugins, 'find').mockReturnValue({
          key: 'test-plugin',
          action: mockPluginAction,
        } as any);

        await synthesize({
          numTests: 1,
          plugins: [{ id: 'test-plugin', numTests: 1 }],
          prompts: ['Test prompt'],
          strategies: [{ id: 'jailbreak:meta' }],
          targetIds: ['test-provider'],
        });

        // Verify the policy parameter matches expected
        expect(mockExtractGoal).toHaveBeenCalled();
        const call = mockExtractGoal.mock.calls[0];
        expect(call[3]).toBe(testCase.expectedPolicy);
      }
    });
  });

  describe('strategy numTests capping integration', () => {
    beforeEach(() => {
      vi.clearAllMocks();
      vi.resetAllMocks();

      vi.mocked(logger.info).mockImplementation(() => logger as any);
      vi.mocked(logger.warn).mockImplementation(() => logger as any);
      vi.mocked(logger.debug).mockImplementation(() => logger as any);
      vi.mocked(logger.error).mockImplementation(() => logger as any);
      vi.mocked(extractSystemPurpose).mockResolvedValue('Test purpose');
      vi.mocked(extractEntities).mockResolvedValue([]);
      vi.mocked(shouldGenerateRemote).mockReturnValue(false);
      vi.mocked(checkRemoteHealth).mockResolvedValue({
        status: 'OK',
        message: 'Cloud API is healthy',
      });
      vi.mocked(getRemoteHealthUrl).mockReturnValue('http://test.com/health');
      vi.mocked(extractVariablesFromTemplates).mockReturnValue(['query']);

      vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
        return {
          start: vi.fn(),
          update: vi.fn(),
          stop: vi.fn(),
          increment: vi.fn(),
        } as any;
      });
    });

    it('should cap strategy output when numTests is configured', async () => {
      // Mock plugin to return 10 test cases
      const mockPluginAction = vi.fn().mockResolvedValue(
        Array(10)
          .fill(null)
          .map((_, i) => ({ vars: { query: `test${i}` } })),
      );
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      // Mock strategy that returns all input tests (1:1)
      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((tc: any) => ({
          ...tc,
          metadata: { ...tc.metadata, strategyId: 'base64' },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'base64',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        numTests: 10,
        plugins: [{ id: 'test-plugin', numTests: 10 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'base64', config: { numTests: 3 } }],
        targetIds: ['test-provider'],
      });

      // Basic tests: 10, Strategy tests: 3 (capped from 10)
      const strategyTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'base64');
      expect(strategyTests.length).toBe(3);
    });

    it('should log warning when numTests is 0', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((tc: any) => ({
          ...tc,
          metadata: { ...tc.metadata, strategyId: 'base64' },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'base64',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'base64', config: { numTests: 0 } }],
        targetIds: ['test-provider'],
      });

      // Should have 0 strategy tests
      const strategyTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'base64');
      expect(strategyTests.length).toBe(0);

      // Should have logged warning
      expect(logger.warn).toHaveBeenCalledWith(
        expect.stringContaining('numTests=0 configured, skipping strategy'),
      );
    });

    it('should log debug when capping occurs', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue(
        Array(5)
          .fill(null)
          .map((_, i) => ({ vars: { query: `test${i}` } })),
      );
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((tc: any) => ({
          ...tc,
          metadata: { ...tc.metadata, strategyId: 'base64' },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'base64',
        action: mockStrategyAction,
      });

      await synthesize({
        numTests: 5,
        plugins: [{ id: 'test-plugin', numTests: 5 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'base64', config: { numTests: 2 } }],
        targetIds: ['test-provider'],
      });

      // Should have logged debug about pre-limiting
      expect(logger.debug).toHaveBeenCalledWith(
        expect.stringContaining('Pre-limiting 5 tests to numTests=2'),
      );
    });

    it('should not cap when numTests is undefined', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue(
        Array(5)
          .fill(null)
          .map((_, i) => ({ vars: { query: `test${i}` } })),
      );
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((tc: any) => ({
          ...tc,
          metadata: { ...tc.metadata, strategyId: 'base64' },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'base64',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        numTests: 5,
        plugins: [{ id: 'test-plugin', numTests: 5 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'base64' }], // No numTests config
        targetIds: ['test-provider'],
      });

      // All 5 strategy tests should be present (no capping)
      const strategyTests = result.testCases.filter((tc) => tc.metadata?.strategyId === 'base64');
      expect(strategyTests.length).toBe(5);
    });

    it('should pass pre-limited tests to strategy (not all tests)', async () => {
      // This is the key test: verify strategy receives limited input, not all tests
      const mockPluginAction = vi.fn().mockResolvedValue(
        Array(10)
          .fill(null)
          .map((_, i) => ({ vars: { query: `test${i}` } })),
      );
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((tc: any) => ({
          ...tc,
          metadata: { ...tc.metadata, strategyId: 'base64' },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'base64',
        action: mockStrategyAction,
      });

      await synthesize({
        numTests: 10,
        plugins: [{ id: 'test-plugin', numTests: 10 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'base64', config: { numTests: 3 } }],
        targetIds: ['test-provider'],
      });

      // KEY ASSERTION: Strategy should receive only 3 tests, not all 10
      // This verifies pre-limiting works (avoids wasted computation)
      expect(mockStrategyAction).toHaveBeenCalled();
      const receivedTests = mockStrategyAction.mock.calls[0][0];
      expect(receivedTests.length).toBe(3);
    });

    it('should not call strategy when numTests is 0', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.map((tc: any) => ({
          ...tc,
          metadata: { ...tc.metadata, strategyId: 'base64' },
        })),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'base64',
        action: mockStrategyAction,
      });

      await synthesize({
        numTests: 1,
        plugins: [{ id: 'test-plugin', numTests: 1 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'base64', config: { numTests: 0 } }],
        targetIds: ['test-provider'],
      });

      // Strategy should NOT be called when numTests=0 (early exit)
      expect(mockStrategyAction).not.toHaveBeenCalled();
    });

    it('should apply post-cap safety net for 1:N fan-out strategies', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue(
        Array(5)
          .fill(null)
          .map((_, i) => ({ vars: { query: `test${i}` } })),
      );
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'test-plugin',
      });

      // Mock strategy that generates 2x the input (1:2 fan-out)
      const mockStrategyAction = vi.fn().mockImplementation((testCases) =>
        testCases.flatMap((tc: any) => [
          { ...tc, metadata: { ...tc.metadata, strategyId: 'multilingual', variant: 'a' } },
          { ...tc, metadata: { ...tc.metadata, strategyId: 'multilingual', variant: 'b' } },
        ]),
      );
      vi.spyOn(Strategies, 'find').mockReturnValue({
        id: 'multilingual',
        action: mockStrategyAction,
      });

      const result = await synthesize({
        numTests: 5,
        plugins: [{ id: 'test-plugin', numTests: 5 }],
        prompts: ['Test prompt'],
        strategies: [{ id: 'multilingual', config: { numTests: 3 } }],
        targetIds: ['test-provider'],
      });

      // Strategy receives 3 tests (pre-limit), generates 6 (2x fan-out), capped to 3 (post-cap)
      const strategyTests = result.testCases.filter(
        (tc) => tc.metadata?.strategyId === 'multilingual',
      );
      expect(strategyTests.length).toBe(3);

      // Should have logged warning about post-cap safety net
      expect(logger.warn).toHaveBeenCalledWith(
        expect.stringContaining('Post-cap safety net applied'),
      );
    });
  });

  describe('plugin logging and report aggregation', () => {
    beforeEach(() => {
      vi.clearAllMocks();
      vi.resetAllMocks();

      vi.mocked(logger.info).mockImplementation(() => logger as any);
      vi.mocked(logger.warn).mockImplementation(() => logger as any);
      vi.mocked(logger.debug).mockImplementation(() => logger as any);
      vi.mocked(logger.error).mockImplementation(() => logger as any);
      vi.mocked(extractSystemPurpose).mockResolvedValue('Test purpose');
      vi.mocked(extractEntities).mockResolvedValue([]);
      vi.mocked(extractVariablesFromTemplates).mockReturnValue(['query']);
      vi.mocked(checkRemoteHealth).mockResolvedValue({
        status: 'OK',
        message: 'Cloud API is healthy',
      });
      vi.mocked(getRemoteHealthUrl).mockReturnValue('http://health.test');
      vi.mocked(shouldGenerateRemote).mockReturnValue(true);
    });

    it('should show truncated policy text in plugin list for policy plugins', async () => {
      const longPolicyText =
        'The assistant must not reveal any internal implementation details such as schema definitions, parameter lists, tool routing logic.';
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      await synthesize({
        numTests: 1,
        plugins: [{ id: 'policy', numTests: 1, config: { policy: longPolicyText } }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Check that the "Using plugins:" log contains truncated policy text
      const pluginListMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find((arg): arg is string => typeof arg === 'string' && arg.includes('Using plugins:'));

      expect(pluginListMessage).toBeDefined();
      // Should contain truncated text (70 chars + ...)
      expect(pluginListMessage).toContain(
        'The assistant must not reveal any internal implementation details such',
      );
      expect(pluginListMessage).toContain('...');
      // Should NOT contain full JSON config
      expect(pluginListMessage).not.toContain('"policy":');
    });

    it('should log config shape at debug level for policy plugins', async () => {
      const policyText = 'SECRET_POLICY_DEBUG_TEXT';
      const secretConfigKey = 'SECRET_CONFIG_KEY_NAME';
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      await synthesize({
        numTests: 1,
        plugins: [
          { id: 'policy', numTests: 1, config: { policy: policyText, [secretConfigKey]: true } },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(logger.debug).toHaveBeenCalledWith('Plugin config', {
        pluginId: 'policy',
        configKeyCount: 2,
      });
      const debugLogs = JSON.stringify(vi.mocked(logger.debug).mock.calls);
      expect(debugLogs).not.toContain(policyText);
      expect(debugLogs).not.toContain(secretConfigKey);
    });

    it('should show "(custom config)" for non-policy plugins with config', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'harmful:hate',
      });

      await synthesize({
        numTests: 1,
        plugins: [{ id: 'harmful:hate', numTests: 1, config: { someOption: 'value' } }],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      const pluginListMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find((arg): arg is string => typeof arg === 'string' && arg.includes('Using plugins:'));

      expect(pluginListMessage).toBeDefined();
      expect(pluginListMessage).toContain('(custom config)');
      // Should NOT contain full JSON
      expect(pluginListMessage).not.toContain('someOption');
    });

    it('should not log non-policy plugin config keys or values at debug level', async () => {
      const secretConfigKey = 'SECRET_PLUGIN_CONFIG_KEY';
      const secretConfigValue = 'SECRET_PLUGIN_CONFIG_VALUE';
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'harmful:hate',
      });

      await synthesize({
        numTests: 1,
        plugins: [
          {
            id: 'harmful:hate',
            numTests: 1,
            config: { [secretConfigKey]: secretConfigValue, severity: 'high' },
          },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      expect(logger.debug).toHaveBeenCalledWith('Plugin config', {
        pluginId: 'harmful:hate',
        configKeyCount: 2,
      });
      const debugLogs = JSON.stringify(vi.mocked(logger.debug).mock.calls);
      expect(debugLogs).not.toContain(secretConfigKey);
      expect(debugLogs).not.toContain(secretConfigValue);
    });

    it('should show separate rows for multiple policy plugins with unique IDs', async () => {
      const mockPluginAction = vi
        .fn()
        .mockResolvedValue([{ vars: { query: 'test1' } }, { vars: { query: 'test2' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      await synthesize({
        numTests: 2,
        plugins: [
          { id: 'policy', numTests: 2, config: { policy: 'Policy 1' } },
          { id: 'policy', numTests: 2, config: { policy: 'Policy 2' } },
          { id: 'policy', numTests: 2, config: { policy: 'Policy 3' } },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      // Find the report in logger.info calls
      const reportMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find(
          (arg): arg is string => typeof arg === 'string' && arg.includes('Test Generation Report'),
        );

      expect(reportMessage).toBeDefined();
      // Strip ANSI codes for easier assertion
      const cleanReport = stripAnsi(reportMessage || '');

      // Each policy plugin should have its own row with display format "policy [hash]: preview..."
      // Inline policies show hash and preview
      expect(cleanReport).toMatch(/policy \[[a-f0-9]{12}\]:/);
      // Count unique policy rows (should be 3)
      const policyMatches = cleanReport.match(/policy \[[a-f0-9]{12}\]:/g);
      expect(policyMatches?.length).toBe(3);
      // Each should show 2 requested, 2 generated
      const twoMatches = cleanReport.match(/\b2\b/g);
      expect(twoMatches?.length).toBeGreaterThanOrEqual(6); // At least 6 occurrences of "2"
    });

    it('should show separate rows for each policy and language combination', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      await synthesize({
        numTests: 1,
        language: ['Hmong', 'Zulu'],
        plugins: [
          { id: 'policy', numTests: 1, config: { policy: 'Policy 1' } },
          { id: 'policy', numTests: 1, config: { policy: 'Policy 2' } },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      const reportMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find(
          (arg): arg is string => typeof arg === 'string' && arg.includes('Test Generation Report'),
        );

      expect(reportMessage).toBeDefined();
      const cleanReport = stripAnsi(reportMessage || '');

      // Each policy should have separate rows for each language
      // Display format: "(Lang) policy [hash]: preview..."
      const hmongMatches = cleanReport.match(/\(Hmong\) policy \[[a-f0-9]{12}\]:/g);
      const zuluMatches = cleanReport.match(/\(Zulu\) policy \[[a-f0-9]{12}\]:/g);
      expect(hmongMatches?.length).toBe(2); // 2 policies in Hmong
      expect(zuluMatches?.length).toBe(2); // 2 policies in Zulu
      // Each should show 1 requested, 1 generated
      const oneMatches = cleanReport.match(/\b1\b/g);
      expect(oneMatches?.length).toBeGreaterThanOrEqual(8); // At least 8 occurrences of "1"
    });

    it('should use policy name when available instead of hash + truncated text', async () => {
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockReturnValue({
        action: mockPluginAction,
        key: 'policy',
      });

      await synthesize({
        numTests: 2,
        plugins: [
          // Policy with a name - should display the name
          {
            id: 'policy',
            numTests: 2,
            config: {
              policy: {
                id: 'abc123def456',
                text: 'Some policy text',
                name: 'Secret Protection Policy',
              },
            },
          },
          // Policy without a name - should display hash + truncated text
          { id: 'policy', numTests: 2, config: { policy: 'Another policy without a name' } },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      const reportMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find(
          (arg): arg is string => typeof arg === 'string' && arg.includes('Test Generation Report'),
        );

      expect(reportMessage).toBeDefined();
      const cleanReport = stripAnsi(reportMessage || '');

      // Named policy should show just the name (no hash in display)
      expect(cleanReport).toMatch(/Secret Protection Policy/);
      expect(cleanReport).not.toMatch(/Secret Protection Policy \[[a-f0-9]/); // No hash after name
      // Inline policy should show: "policy [hash]: preview..."
      expect(cleanReport).toMatch(/policy \[[a-f0-9]{12}\]:/);
    });

    it('should work correctly with both built-in plugins and policy plugins', async () => {
      // Mock different plugins
      const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
      vi.spyOn(Plugins, 'find').mockImplementation(function (predicate) {
        const mockPlugins = [
          { key: 'policy', action: mockPluginAction },
          { key: 'hallucination', action: mockPluginAction },
          { key: 'contracts', action: mockPluginAction },
        ];
        if (typeof predicate === 'function') {
          return mockPlugins.find(predicate);
        }
        return undefined;
      });

      await synthesize({
        numTests: 2,
        plugins: [
          // Built-in plugin - hallucination
          { id: 'hallucination', numTests: 2 },
          // Built-in plugin - contracts
          { id: 'contracts', numTests: 2 },
          // Policy plugin with name (cloud-style)
          {
            id: 'policy',
            numTests: 2,
            config: {
              policy: {
                id: 'abc123def456',
                text: 'Never share confidential data',
                name: 'Data Protection Policy',
              },
            },
          },
          // Policy plugin without name (inline)
          { id: 'policy', numTests: 2, config: { policy: 'Always be respectful to users' } },
        ],
        prompts: ['Test prompt'],
        strategies: [],
        targetIds: ['test-provider'],
      });

      const reportMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find(
          (arg): arg is string => typeof arg === 'string' && arg.includes('Test Generation Report'),
        );

      expect(reportMessage).toBeDefined();
      const cleanReport = stripAnsi(reportMessage || '');

      // Built-in plugins should show their ID directly
      expect(cleanReport).toMatch(/hallucination/);
      expect(cleanReport).toMatch(/contracts/);
      // Named policy should show just the name
      expect(cleanReport).toMatch(/Data Protection Policy/);
      expect(cleanReport).not.toMatch(/Data Protection Policy \[/); // No ID after name
      // Inline policy should show "policy [hash]: preview..."
      expect(cleanReport).toMatch(/policy \[[a-f0-9]{12}\]:/);
      // Should have 4 plugin rows (hallucination, contracts, named policy, inline policy)
      const pluginRows = cleanReport.match(/│\s+\d+\s+│\s+Plugin/g);
      expect(pluginRows?.length).toBe(4);
    });
  });

  describe('redteamProvider config propagation to strategies', () => {
    beforeEach(() => {
      vi.resetAllMocks();

      vi.mocked(logger.info).mockImplementation(() => logger as any);
      vi.mocked(logger.warn).mockImplementation(() => logger as any);
      vi.mocked(logger.debug).mockImplementation(() => logger as any);
      vi.mocked(logger.error).mockImplementation(() => logger as any);
      vi.mocked(extractSystemPurpose).mockResolvedValue('Test purpose');
      vi.mocked(extractEntities).mockResolvedValue([]);
      vi.mocked(shouldGenerateRemote).mockReturnValue(false);
      vi.mocked(checkRemoteHealth).mockResolvedValue({
        status: 'OK',
        message: 'Cloud API is healthy',
      });
      vi.mocked(getRemoteHealthUrl).mockReturnValue('http://test.com/health');
      vi.mocked(extractVariablesFromTemplates).mockReturnValue(['query']);

      vi.mocked(cliProgress.SingleBar).mockImplementation(function () {
        return {
          start: vi.fn(),
          update: vi.fn(),
          stop: vi.fn(),
          increment: vi.fn(),
        } as any;
      });
    });

    it('should pass redteamProvider from cliState.config to strategy actions', async () => {
      // Import cliState to set up the redteam provider config
      const cliState = (await import('../../src/cliState')).default;
      const originalConfig = cliState.config;

      // Set up cliState with a mock redteam provider - this is the provider that should
      // be passed to strategies for use by agentic providers (iterative, crescendo, etc.)
      cliState.config = {
        redteam: {
          provider: 'vertex:gemini-2.5-flash',
        },
      } as any;

      try {
        // Mock plugin to return a test case
        const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
        vi.spyOn(Plugins, 'find').mockReturnValue({
          action: mockPluginAction,
          key: 'test-plugin',
        });

        // Mock strategy that captures the config it receives
        let capturedConfig: Record<string, any> | undefined;
        const mockStrategyAction = vi.fn().mockImplementation((testCases, _injectVar, config) => {
          capturedConfig = config;
          return testCases.map((tc: any) => ({
            ...tc,
            metadata: { ...tc.metadata, strategyId: 'jailbreak' },
          }));
        });

        vi.spyOn(Strategies, 'find').mockReturnValue({
          id: 'jailbreak',
          action: mockStrategyAction,
        });

        // Mock the provider loading to return a mock provider for test generation
        // This avoids the loadApiProviders error while still testing strategy config
        const mockProvider = { id: () => 'mock-provider', callApi: vi.fn() };
        const providersShared = await import('../../src/redteam/providers/shared');
        const getProviderSpy = vi
          .spyOn(providersShared.redteamProviderManager, 'getProvider')
          .mockResolvedValue(mockProvider as any);
        const getGradingProviderSpy = vi
          .spyOn(providersShared.redteamProviderManager, 'getGradingProvider')
          .mockResolvedValue(mockProvider as any);
        const getMultilingualProviderSpy = vi
          .spyOn(providersShared.redteamProviderManager, 'getMultilingualProvider')
          .mockResolvedValue(undefined);

        try {
          await synthesize({
            cloudTargetDatabaseId: 'cloud-target-123',
            numTests: 1,
            plugins: [{ id: 'test-plugin', numTests: 1 }],
            prompts: ['Test prompt'],
            strategies: [{ id: 'jailbreak' }],
            targetIds: ['test-provider'],
          });

          // KEY ASSERTION: The strategy should receive redteamProvider from cliState.config
          expect(mockStrategyAction).toHaveBeenCalled();
          expect(capturedConfig).toBeDefined();
          expect(capturedConfig?.redteamProvider).toBe('vertex:gemini-2.5-flash');
          expect(capturedConfig?.targetId).toBe('cloud-target-123');
        } finally {
          getProviderSpy.mockRestore();
          getGradingProviderSpy.mockRestore();
          getMultilingualProviderSpy.mockRestore();
        }
      } finally {
        // Restore original cliState
        cliState.config = originalConfig;
      }
    });

    it('should pass redteamProvider as undefined when not configured in cliState', async () => {
      const cliState = (await import('../../src/cliState')).default;
      const originalConfig = cliState.config;

      // Set up cliState WITHOUT redteam provider
      cliState.config = {
        redteam: {},
      } as any;

      try {
        const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
        vi.spyOn(Plugins, 'find').mockReturnValue({
          action: mockPluginAction,
          key: 'test-plugin',
        });

        let capturedConfig: Record<string, any> | undefined;
        const mockStrategyAction = vi.fn().mockImplementation((testCases, _injectVar, config) => {
          capturedConfig = config;
          return testCases.map((tc: any) => ({
            ...tc,
            metadata: { ...tc.metadata, strategyId: 'jailbreak' },
          }));
        });

        vi.spyOn(Strategies, 'find').mockReturnValue({
          id: 'jailbreak',
          action: mockStrategyAction,
        });

        await synthesize({
          numTests: 1,
          plugins: [{ id: 'test-plugin', numTests: 1 }],
          prompts: ['Test prompt'],
          strategies: [{ id: 'jailbreak' }],
          targetIds: ['test-provider'],
        });

        expect(mockStrategyAction).toHaveBeenCalled();
        expect(capturedConfig).toBeDefined();
        // When not configured, redteamProvider should be undefined
        expect(capturedConfig?.redteamProvider).toBeUndefined();
      } finally {
        cliState.config = originalConfig;
      }
    });

    it('should pass redteamProvider as object when configured as provider options', async () => {
      const cliState = (await import('../../src/cliState')).default;
      const originalConfig = cliState.config;

      // Set up cliState with provider options object
      const providerOptions = {
        id: 'vertex:gemini-2.5-flash',
        config: { temperature: 0.7 },
      };
      cliState.config = {
        redteam: {
          provider: providerOptions,
        },
      } as any;

      try {
        const mockPluginAction = vi.fn().mockResolvedValue([{ vars: { query: 'test' } }]);
        vi.spyOn(Plugins, 'find').mockReturnValue({
          action: mockPluginAction,
          key: 'test-plugin',
        });

        let capturedConfig: Record<string, any> | undefined;
        const mockStrategyAction = vi.fn().mockImplementation((testCases, _injectVar, config) => {
          capturedConfig = config;
          return testCases.map((tc: any) => ({
            ...tc,
            metadata: { ...tc.metadata, strategyId: 'jailbreak' },
          }));
        });

        vi.spyOn(Strategies, 'find').mockReturnValue({
          id: 'jailbreak',
          action: mockStrategyAction,
        });

        // Mock the provider loading
        const mockProvider = { id: () => 'mock-provider', callApi: vi.fn() };
        const providersShared = await import('../../src/redteam/providers/shared');
        const getProviderSpy = vi
          .spyOn(providersShared.redteamProviderManager, 'getProvider')
          .mockResolvedValue(mockProvider as any);
        const getGradingProviderSpy = vi
          .spyOn(providersShared.redteamProviderManager, 'getGradingProvider')
          .mockResolvedValue(mockProvider as any);
        const getMultilingualProviderSpy = vi
          .spyOn(providersShared.redteamProviderManager, 'getMultilingualProvider')
          .mockResolvedValue(undefined);

        try {
          await synthesize({
            numTests: 1,
            plugins: [{ id: 'test-plugin', numTests: 1 }],
            prompts: ['Test prompt'],
            strategies: [{ id: 'jailbreak' }],
            targetIds: ['test-provider'],
          });

          expect(mockStrategyAction).toHaveBeenCalled();
          expect(capturedConfig).toBeDefined();
          // Should pass the full provider options object
          expect(capturedConfig?.redteamProvider).toEqual(providerOptions);
        } finally {
          getProviderSpy.mockRestore();
          getGradingProviderSpy.mockRestore();
          getMultilingualProviderSpy.mockRestore();
        }
      } finally {
        cliState.config = originalConfig;
      }
    });
  });

  describe('Multi-input mode plugin exclusion', () => {
    it('should exclude dataset-exempt plugins in multi-input mode', async () => {
      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'aegis', numTests: 1 }, // DATASET_EXEMPT_PLUGIN
          { id: 'contracts', numTests: 1 }, // Regular plugin
        ],
        prompts: ['Test {{query}}'],
        strategies: [],
        targetIds: ['test-provider'],
        inputs: { query: 'user query', context: 'additional context' }, // Multi-input mode
      });

      // aegis should be excluded, contracts should remain
      // Result should only contain test cases from contracts
      expect(result.testCases.length).toBeGreaterThanOrEqual(0);

      // Check that logger.info was called with skipping message
      const skipMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find((arg): arg is string => typeof arg === 'string' && arg.includes('Skipping'));

      expect(skipMessage).toBeDefined();
      expect(skipMessage).toContain('aegis');
    });

    it('should exclude multi-input excluded plugins in multi-input mode', async () => {
      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'cca', numTests: 1 }, // MULTI_INPUT_EXCLUDED_PLUGIN
          { id: 'cross-session-leak', numTests: 1 }, // MULTI_INPUT_EXCLUDED_PLUGIN
          { id: 'contracts', numTests: 1 }, // Regular plugin
        ],
        prompts: ['Test {{query}}'],
        strategies: [],
        targetIds: ['test-provider'],
        inputs: { query: 'user query', context: 'additional context' }, // Multi-input mode
      });

      // cca and cross-session-leak should be excluded
      expect(result.testCases.length).toBeGreaterThanOrEqual(0);

      // Check that logger.info was called with skipping message
      const skipMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find((arg): arg is string => typeof arg === 'string' && arg.includes('Skipping'));

      expect(skipMessage).toBeDefined();
      expect(skipMessage).toContain('cca');
      expect(skipMessage).toContain('cross-session-leak');
    });

    it('should NOT exclude plugins when inputs is empty object', async () => {
      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'cca', numTests: 1 }, // Would be excluded in multi-input mode
          { id: 'contracts', numTests: 1 },
        ],
        prompts: ['Test {{query}}'],
        strategies: [],
        targetIds: ['test-provider'],
        inputs: {}, // Empty inputs - not multi-input mode
      });

      // No plugins should be excluded
      expect(result.testCases.length).toBeGreaterThanOrEqual(0);

      // Should NOT have skipping message for cca
      const skipMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find(
          (arg): arg is string =>
            typeof arg === 'string' && arg.includes('Skipping') && arg.includes('cca'),
        );

      expect(skipMessage).toBeUndefined();
    });

    it('should NOT exclude plugins when inputs is undefined', async () => {
      const result = await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'cca', numTests: 1 }, // Would be excluded in multi-input mode
          { id: 'contracts', numTests: 1 },
        ],
        prompts: ['Test {{query}}'],
        strategies: [],
        targetIds: ['test-provider'],
        // No inputs - not multi-input mode
      });

      // No plugins should be excluded
      expect(result.testCases.length).toBeGreaterThanOrEqual(0);

      // Should NOT have skipping message for cca
      const skipMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find(
          (arg): arg is string =>
            typeof arg === 'string' && arg.includes('Skipping') && arg.includes('cca'),
        );

      expect(skipMessage).toBeUndefined();
    });

    it('should log info about using multi-input mode', async () => {
      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [{ id: 'contracts', numTests: 1 }],
        prompts: ['Test {{query}}'],
        strategies: [],
        targetIds: ['test-provider'],
        inputs: { query: 'user query', context: 'additional context' },
      });

      // Check that logger.info was called with multi-input mode message
      const multiInputMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find((arg): arg is string => typeof arg === 'string' && arg.includes('multi-input mode'));

      expect(multiInputMessage).toBeDefined();
      expect(multiInputMessage).toContain('2 variables');
      expect(multiInputMessage).toContain('query');
      expect(multiInputMessage).toContain('context');
    });

    it('should exclude only plugins that still do not support multi-input mode', async () => {
      await synthesize({
        language: 'en',
        numTests: 1,
        plugins: [
          { id: 'cca', numTests: 1 },
          { id: 'cross-session-leak', numTests: 1 },
          { id: 'special-token-injection', numTests: 1 },
          { id: 'system-prompt-override', numTests: 1 },
          { id: 'ascii-smuggling', numTests: 1 },
          { id: 'contracts', numTests: 1 }, // Regular plugin - should NOT be excluded
        ],
        prompts: ['Test {{query}}'],
        strategies: [],
        targetIds: ['test-provider'],
        inputs: { query: 'user query', context: 'additional context' },
      });

      // Check that only the explicit non-dataset exclusions are in the skip message.
      const skipMessage = vi
        .mocked(logger.info)
        .mock.calls.map(([arg]) => arg)
        .find((arg): arg is string => typeof arg === 'string' && arg.includes('Skipping 2 plugin'));

      expect(skipMessage).toBeDefined();
      expect(skipMessage).toContain('cca');
      expect(skipMessage).toContain('cross-session-leak');
      expect(skipMessage).not.toContain('special-token-injection');
      expect(skipMessage).not.toContain('system-prompt-override');
      expect(skipMessage).not.toContain('ascii-smuggling');
    });
  });
});