import { beforeEach, describe, expect, it, vi } from 'vitest'; import { BIAS_PLUGINS } from '../../../src/redteam/constants'; import { BiasGrader } from '../../../src/redteam/plugins/bias'; import { Plugins } from '../../../src/redteam/plugins/index'; import { getRemoteGenerationExplicitlyDisabledError, neverGenerateRemote, } from '../../../src/redteam/remoteGeneration'; import { createMockProvider, type MockApiProvider } from '../../factories/provider'; import type { PluginActionParams } from '../../../src/types/index'; vi.mock('../../../src/redteam/remoteGeneration'); describe('Bias Plugin', () => { let mockProvider: MockApiProvider; let mockPluginParams: PluginActionParams; beforeEach(() => { mockProvider = createMockProvider(); mockPluginParams = { provider: mockProvider, purpose: 'test purpose', injectVar: 'testVar', n: 3, config: {}, delayMs: 0, }; vi.mocked(neverGenerateRemote).mockImplementation(function () { return false; }); vi.mocked(getRemoteGenerationExplicitlyDisabledError).mockImplementation( (strategyName) => `${strategyName} requires remote generation (explicitly disabled).`, ); }); describe('remote-only behavior', () => { it('should return empty array when remote generation is disabled for age bias', async () => { vi.mocked(neverGenerateRemote).mockImplementation(function () { return true; }); const agebiasPlugin = Plugins.find((p) => p.key === 'bias:age'); expect(agebiasPlugin).toBeDefined(); const result = await agebiasPlugin!.action(mockPluginParams); expect(result).toEqual([]); }); it('should return empty array when remote generation is disabled for disability bias', async () => { vi.mocked(neverGenerateRemote).mockImplementation(function () { return true; }); const disabilityBiasPlugin = Plugins.find((p) => p.key === 'bias:disability'); expect(disabilityBiasPlugin).toBeDefined(); const result = await disabilityBiasPlugin!.action(mockPluginParams); expect(result).toEqual([]); }); it('should return empty array when remote generation is disabled for gender bias', async () => { vi.mocked(neverGenerateRemote).mockImplementation(function () { return true; }); const genderBiasPlugin = Plugins.find((p) => p.key === 'bias:gender'); expect(genderBiasPlugin).toBeDefined(); const result = await genderBiasPlugin!.action(mockPluginParams); expect(result).toEqual([]); }); it('should return empty array when remote generation is disabled for race bias', async () => { vi.mocked(neverGenerateRemote).mockImplementation(function () { return true; }); const raceBiasPlugin = Plugins.find((p) => p.key === 'bias:race'); expect(raceBiasPlugin).toBeDefined(); const result = await raceBiasPlugin!.action(mockPluginParams); expect(result).toEqual([]); }); }); describe('BiasGrader', () => { let grader: BiasGrader; beforeEach(() => { grader = new BiasGrader(); }); it('should have correct plugin ID', () => { expect(grader.id).toBe('promptfoo:redteam:bias'); }); it('should have bias-specific rubric content', () => { expect(grader.rubric).toContain('bias'); expect(grader.rubric).toContain('stereotyping'); expect(grader.rubric).toContain('discrimination'); expect(grader.rubric).toContain('protected characteristics'); }); it('should use standardized purpose tags in rubric', () => { expect(grader.rubric).toContain(''); expect(grader.rubric).toContain(''); expect(grader.rubric).not.toContain(''); expect(grader.rubric).not.toContain(''); }); it('should include evaluation criteria in rubric', () => { expect(grader.rubric).toContain('FAIL'); expect(grader.rubric).toContain('PASS'); expect(grader.rubric).toContain('Stereotyping'); expect(grader.rubric).toContain('Discriminatory Language'); expect(grader.rubric).toContain('Unfair Treatment'); }); it('should include guidance for avoiding false positives', () => { expect(grader.rubric).toContain('false positives'); expect(grader.rubric).toContain('accessibility'); expect(grader.rubric).toContain('accommodations'); }); }); describe('Plugin Constants', () => { it('should include all expected bias plugin types', () => { expect(BIAS_PLUGINS).toContain('bias:age'); expect(BIAS_PLUGINS).toContain('bias:disability'); expect(BIAS_PLUGINS).toContain('bias:gender'); expect(BIAS_PLUGINS).toContain('bias:race'); expect(BIAS_PLUGINS).toHaveLength(4); }); }); });