chore: import upstream snapshot with attribution

test/server/services/evalJobService.test.ts

@@ -0,0 +1,136 @@
+import { describe, expect, it } from 'vitest';
+import { EvalJobService } from '../../../src/server/services/evalJobService';
+
+describe('EvalJobService', () => {
+  it('creates an in-progress job', () => {
+    const service = new EvalJobService();
+
+    expect(service.create('job-1')).toEqual({
+      evalId: null,
+      status: 'in-progress',
+      progress: 0,
+      total: 0,
+      result: null,
+      logs: [],
+    });
+  });
+
+  it('tracks progress, completion, and logs', () => {
+    const service = new EvalJobService();
+    service.create('job-1');
+
+    expect(service.setProgress('job-1', 2, 5)).toBe(true);
+    expect(service.appendLog('job-1', 'working')).toBe(true);
+    expect(service.complete('job-1', { results: [] } as never, 'eval-1')).toBe(true);
+
+    expect(service.get('job-1')).toMatchObject({
+      evalId: 'eval-1',
+      status: 'complete',
+      progress: 2,
+      total: 5,
+      result: { results: [] },
+      logs: ['working'],
+    });
+  });
+
+  it('returns defensive snapshots', () => {
+    const service = new EvalJobService();
+    service.create('job-1');
+    const result = { results: [{ output: { nested: 'original' } }] } as never;
+    service.complete('job-1', result, 'eval-1');
+
+    const snapshot = service.get('job-1');
+    expect(snapshot).toBeDefined();
+    snapshot?.logs.push('mutated outside service');
+    (snapshot?.result as any).results[0].output.nested = 'mutated outside service';
+    (result as any).results[0].output.nested = 'mutated after completion';
+
+    expect(service.get('job-1')?.logs).toEqual([]);
+    expect((service.get('job-1')?.result as any).results[0].output.nested).toBe('original');
+  });
+
+  it('stores JSON-safe snapshots for function-backed prompts', () => {
+    const service = new EvalJobService();
+    service.create('job-1');
+
+    expect(() =>
+      service.complete(
+        'job-1',
+        {
+          prompts: [{ function: () => 'generated prompt', label: 'dynamic prompt' }],
+          results: [],
+        } as never,
+        'eval-1',
+      ),
+    ).not.toThrow();
+    expect(service.get('job-1')?.result).toEqual({
+      prompts: [{ label: 'dynamic prompt' }],
+      results: [],
+    });
+  });
+
+  it('stores JSON-safe snapshots for circular references and bigint values', () => {
+    const service = new EvalJobService();
+    service.create('job-1');
+    const output: { count: bigint; self?: unknown } = { count: 42n };
+    output.self = output;
+
+    expect(() =>
+      service.complete(
+        'job-1',
+        {
+          results: [{ response: { output } }],
+        } as never,
+        'eval-1',
+      ),
+    ).not.toThrow();
+    expect(service.get('job-1')?.result).toEqual({
+      results: [{ response: { output: { count: '42' } } }],
+    });
+  });
+
+  it('supports replacing and appending failure logs', () => {
+    const service = new EvalJobService();
+    service.create('job-1');
+    service.appendLog('job-1', 'before failure');
+
+    expect(service.fail('job-1', ['first failure'])).toBe(true);
+    expect(service.get('job-1')?.logs).toEqual(['first failure']);
+
+    expect(service.fail('job-1', ['second failure'], { append: true })).toBe(true);
+    expect(service.get('job-1')).toMatchObject({
+      evalId: null,
+      status: 'error',
+      result: null,
+      logs: ['first failure', 'second failure'],
+    });
+  });
+
+  it('can preserve completed results when marking a job as failed', () => {
+    const service = new EvalJobService();
+    service.create('job-1');
+    service.complete('job-1', { results: [] } as never, 'eval-1');
+
+    expect(
+      service.fail('job-1', ['cancelled after completion'], {
+        append: true,
+        resetResult: false,
+      }),
+    ).toBe(true);
+    expect(service.get('job-1')).toMatchObject({
+      evalId: 'eval-1',
+      status: 'error',
+      result: { results: [] },
+      logs: ['cancelled after completion'],
+    });
+  });
+
+  it('returns false when updating a missing job', () => {
+    const service = new EvalJobService();
+
+    expect(service.setProgress('missing', 1, 1)).toBe(false);
+    expect(service.complete('missing', null, null)).toBe(false);
+    expect(service.fail('missing', ['missing'])).toBe(false);
+    expect(service.appendLog('missing', 'missing')).toBe(false);
+  });
+});

test/server/services/promptCacheService.test.ts

@@ -0,0 +1,57 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { PromptCacheService } from '../../../src/server/services/promptCacheService';
+import { getPrompts } from '../../../src/util/database';
+import { createDeferred } from '../../util/utils';
+
+vi.mock('../../../src/util/database', () => ({
+  getPrompts: vi.fn(),
+}));
+
+describe('PromptCacheService', () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('caches prompts until invalidated', async () => {
+    vi.mocked(getPrompts)
+      .mockResolvedValueOnce([{ id: 'first' }] as never)
+      .mockResolvedValueOnce([{ id: 'second' }] as never);
+    const service = new PromptCacheService();
+
+    expect(await service.getAll()).toEqual([{ id: 'first' }]);
+    expect(await service.getAll()).toEqual([{ id: 'first' }]);
+    expect(getPrompts).toHaveBeenCalledOnce();
+
+    service.invalidate();
+
+    expect(await service.getAll()).toEqual([{ id: 'second' }]);
+    expect(getPrompts).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not cache a load that resolves after invalidation', async () => {
+    const pending = createDeferred<Awaited<ReturnType<typeof getPrompts>>>();
+    vi.mocked(getPrompts)
+      .mockReturnValueOnce(pending.promise)
+      .mockResolvedValueOnce([{ id: 'fresh' }] as never);
+    const service = new PromptCacheService();
+
+    const staleRequest = service.getAll();
+    service.invalidate();
+    pending.resolve([{ id: 'stale' }] as never);
+
+    await expect(staleRequest).resolves.toEqual([{ id: 'stale' }]);
+    await expect(service.getAll()).resolves.toEqual([{ id: 'fresh' }]);
+    expect(getPrompts).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries after a rejected load', async () => {
+    vi.mocked(getPrompts)
+      .mockRejectedValueOnce(new Error('database unavailable'))
+      .mockResolvedValueOnce([{ id: 'fresh' }] as never);
+    const service = new PromptCacheService();
+
+    await expect(service.getAll()).rejects.toThrow('database unavailable');
+    await expect(service.getAll()).resolves.toEqual([{ id: 'fresh' }]);
+    expect(getPrompts).toHaveBeenCalledTimes(2);
+  });
+});

test/server/services/redteamTestCaseGenerationService.test.ts

@@ -0,0 +1,149 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { createMockResponse } from '../../util/utils';
+
+import type { MultiTurnPromptParams } from '../../../src/server/services/redteamTestCaseGenerationService';
+
+async function getExpectedRemoteGenerationUrl() {
+  const { getRemoteGenerationUrl } = await vi.importActual<
+    typeof import('../../../src/redteam/remoteGeneration')
+  >('../../../src/redteam/remoteGeneration');
+  return getRemoteGenerationUrl();
+}
+const TEST_REQUEST_TIMEOUT_MS = 300000;
+const MOCKED_MODULES = [
+  '../../../src/util/fetch/index',
+  '../../../src/redteam/remoteGeneration',
+  '../../../src/providers/shared',
+  '../../../src/constants',
+];
+
+function mockRemoteGeneration(responseBody: unknown, rejectWith?: Error) {
+  const fetchWithRetries = rejectWith
+    ? vi.fn().mockRejectedValueOnce(rejectWith)
+    : vi.fn().mockResolvedValueOnce(
+        createMockResponse({
+          body: responseBody,
+        }),
+      );
+
+  vi.doMock('../../../src/util/fetch/index', () => ({
+    fetchWithRetries,
+  }));
+  vi.doMock('../../../src/redteam/remoteGeneration', async () => ({
+    getRemoteGenerationUrl: vi.fn().mockReturnValue(await getExpectedRemoteGenerationUrl()),
+    getRemoteGenerationHeaders: vi.fn((extra) => ({
+      'Content-Type': 'application/json',
+      ...extra,
+    })),
+    neverGenerateRemote: vi.fn().mockReturnValue(false),
+  }));
+  vi.doMock('../../../src/providers/shared', () => ({
+    getRequestTimeoutMs: () => TEST_REQUEST_TIMEOUT_MS,
+  }));
+  vi.doMock('../../../src/constants', () => ({
+    VERSION: '0.0.0-test',
+  }));
+
+  return fetchWithRetries;
+}
+
+async function generatePromptForStrategy(strategyId: MultiTurnPromptParams['strategyId']) {
+  const { generateMultiTurnPrompt } = await import(
+    '../../../src/server/services/redteamTestCaseGenerationService'
+  );
+
+  await generateMultiTurnPrompt({
+    pluginId: 'harmful:hate',
+    strategyId,
+    strategyConfigRecord: {},
+    history: [],
+    turn: 0,
+    maxTurns: 5,
+    baseMetadata: { pluginConfig: {} },
+    generatedPrompt: 'initial prompt',
+    purpose: 'test purpose',
+  });
+}
+
+async function expectTaskRequest(fetchWithRetries: ReturnType<typeof vi.fn>, expectedTask: string) {
+  expect(fetchWithRetries).toHaveBeenCalledTimes(1);
+  const [url, request, timeout] = fetchWithRetries.mock.calls[0]!;
+  const body = JSON.parse(String(request.body));
+
+  expect(url).toBe(await getExpectedRemoteGenerationUrl());
+  expect(request).toMatchObject({
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+  });
+  expect(body.task).toBe(expectedTask);
+  expect(timeout).toBe(TEST_REQUEST_TIMEOUT_MS);
+}
+
+describe('redteamTestCaseGenerationService', () => {
+  beforeEach(() => {
+    vi.resetModules();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+    for (const modulePath of MOCKED_MODULES) {
+      vi.doUnmock(modulePath);
+    }
+    vi.resetModules();
+  });
+
+  describe('multi-turn strategy handlers use fetchWithRetries', () => {
+    it('should call fetchWithRetries with correct parameters for GOAT strategy', async () => {
+      const fetchWithRetries = mockRemoteGeneration({
+        message: { content: 'test prompt' },
+        tokenUsage: { total: 100 },
+      });
+
+      await generatePromptForStrategy('goat');
+
+      await expectTaskRequest(fetchWithRetries, 'goat');
+    });
+
+    it('should propagate remote generation failures', async () => {
+      const remoteError = new Error('remote generation failed');
+      const fetchWithRetries = mockRemoteGeneration(undefined, remoteError);
+
+      await expect(generatePromptForStrategy('goat')).rejects.toThrow('remote generation failed');
+      expect(fetchWithRetries).toHaveBeenCalledTimes(1);
+    });
+
+    it('should call fetchWithRetries with correct parameters for Crescendo strategy', async () => {
+      const fetchWithRetries = mockRemoteGeneration({
+        result: {
+          generatedQuestion: 'test question',
+          lastResponseSummary: 'summary',
+          rationaleBehindJailbreak: 'rationale',
+        },
+      });
+
+      await generatePromptForStrategy('crescendo');
+
+      await expectTaskRequest(fetchWithRetries, 'crescendo');
+    });
+
+    it('should call fetchWithRetries with correct parameters for Hydra strategy', async () => {
+      const fetchWithRetries = mockRemoteGeneration({
+        result: { prompt: 'test prompt' },
+      });
+
+      await generatePromptForStrategy('jailbreak:hydra');
+
+      await expectTaskRequest(fetchWithRetries, 'hydra-decision');
+    });
+
+    it('should call fetchWithRetries with correct parameters for Mischievous User strategy', async () => {
+      const fetchWithRetries = mockRemoteGeneration({
+        result: 'test prompt',
+      });
+
+      await generatePromptForStrategy('mischievous-user');
+
+      await expectTaskRequest(fetchWithRetries, 'mischievous-user-redteam');
+    });
+  });
+});