chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { fetchWithCache } from '../../../src/cache';
|
||||
import { VERSION } from '../../../src/constants';
|
||||
import { DEFAULT_PURPOSE, extractSystemPurpose } from '../../../src/redteam/extraction/purpose';
|
||||
import { getRemoteGenerationUrl } from '../../../src/redteam/remoteGeneration';
|
||||
import {
|
||||
createMockProvider,
|
||||
createProviderResponse,
|
||||
type MockApiProvider,
|
||||
} from '../../factories/provider';
|
||||
import { mockProcessEnv } from '../../util/utils';
|
||||
|
||||
vi.mock('../../../src/logger', () => ({
|
||||
default: {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
vi.mock('../../../src/cache', async (importOriginal) => {
|
||||
return {
|
||||
...(await importOriginal()),
|
||||
fetchWithCache: vi.fn(),
|
||||
};
|
||||
});
|
||||
vi.mock('../../../src/redteam/remoteGeneration', async () => ({
|
||||
...(await vi.importActual('../../../src/redteam/remoteGeneration')),
|
||||
getRemoteGenerationUrl: vi.fn().mockReturnValue('https://api.promptfoo.app/api/v1/task'),
|
||||
}));
|
||||
|
||||
describe('System Purpose Extractor', () => {
|
||||
let provider: MockApiProvider;
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
|
||||
beforeAll(() => {
|
||||
originalEnv = { ...process.env };
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockProcessEnv({ ...originalEnv }, { clear: true });
|
||||
mockProcessEnv({ PROMPTFOO_REMOTE_GENERATION_URL: undefined });
|
||||
provider = createMockProvider({
|
||||
response: createProviderResponse({
|
||||
output: '<Purpose>Extracted system purpose</Purpose>',
|
||||
}),
|
||||
});
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(getRemoteGenerationUrl).mockImplementation(function () {
|
||||
return 'https://api.promptfoo.app/api/v1/task';
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
mockProcessEnv(originalEnv, { clear: true });
|
||||
});
|
||||
|
||||
it('should use remote generation when enabled', async () => {
|
||||
mockProcessEnv({ OPENAI_API_KEY: undefined });
|
||||
mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: 'false' });
|
||||
vi.mocked(fetchWithCache).mockResolvedValue({
|
||||
data: { task: 'purpose', result: 'Remote extracted purpose' },
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
cached: false,
|
||||
});
|
||||
|
||||
const result = await extractSystemPurpose(provider, ['prompt1', 'prompt2'], {
|
||||
providerTargetIds: ['file://local-provider.ts'],
|
||||
cloudTargetId: 'cloud-target-123',
|
||||
});
|
||||
|
||||
expect(result).toBe('Remote extracted purpose');
|
||||
expect(fetchWithCache).toHaveBeenCalledWith(
|
||||
'https://api.promptfoo.app/api/v1/task',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
task: 'purpose',
|
||||
prompts: ['prompt1', 'prompt2'],
|
||||
version: VERSION,
|
||||
email: null,
|
||||
targetId: 'cloud-target-123',
|
||||
}),
|
||||
}),
|
||||
expect.any(Number),
|
||||
'json',
|
||||
);
|
||||
});
|
||||
|
||||
it('should not fall back to local extraction when remote generation fails', async () => {
|
||||
mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: 'false' });
|
||||
const originalOpenaiKey = process.env.OPENAI_API_KEY;
|
||||
mockProcessEnv({ OPENAI_API_KEY: undefined });
|
||||
vi.mocked(fetchWithCache).mockRejectedValue(new Error('Remote generation failed'));
|
||||
const result = await extractSystemPurpose(provider, ['prompt1', 'prompt2']);
|
||||
|
||||
expect(result).toBe('');
|
||||
expect(provider.callApi).not.toHaveBeenCalled();
|
||||
mockProcessEnv({ OPENAI_API_KEY: originalOpenaiKey });
|
||||
});
|
||||
|
||||
it('should use local extraction when remote generation is disabled', async () => {
|
||||
mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: 'true' });
|
||||
|
||||
const result = await extractSystemPurpose(provider, ['prompt']);
|
||||
|
||||
expect(result).toBe('Extracted system purpose');
|
||||
expect(provider.callApi).toHaveBeenCalledWith(expect.stringContaining('prompt'));
|
||||
expect(fetchWithCache).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should extract system purpose when returned without xml tags', async () => {
|
||||
mockProcessEnv({ PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: 'true' });
|
||||
vi.mocked(provider.callApi).mockResolvedValue({ output: 'Extracted system purpose' });
|
||||
|
||||
const result = await extractSystemPurpose(provider, ['prompt1', 'prompt2']);
|
||||
|
||||
expect(result).toBe('Extracted system purpose');
|
||||
});
|
||||
|
||||
it('should return default message for empty prompts array', async () => {
|
||||
const result = await extractSystemPurpose(provider, []);
|
||||
expect(result).toBe(DEFAULT_PURPOSE);
|
||||
});
|
||||
|
||||
it('should return default message for prompts array with only template variable', async () => {
|
||||
const result = await extractSystemPurpose(provider, ['{{prompt}}']);
|
||||
expect(result).toBe(DEFAULT_PURPOSE);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user