chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
+97
View File
@@ -0,0 +1,97 @@
import { describe, expect, it } from 'vitest';
import {
collectBlobHashes,
extractBlobHashesFromString,
extractBlobHashesFromValue,
normalizeBlobHash,
} from '../../src/blobs/blobRefs';
const HASH_A = 'a'.repeat(64);
const HASH_B = 'b'.repeat(64);
const uri = (hash: string) => `promptfoo://blob/${hash}`;
describe('normalizeBlobHash', () => {
it('lowercases the hash', () => {
expect(normalizeBlobHash('ABCDEF')).toBe('abcdef');
});
});
describe('extractBlobHashesFromString', () => {
it('returns an empty array for strings without a blob URI', () => {
expect(extractBlobHashesFromString('just some text')).toEqual([]);
});
it('extracts and normalizes every blob hash in the string', () => {
const value = `before ${uri(HASH_A.toUpperCase())} middle ${uri(HASH_B)} after`;
expect(extractBlobHashesFromString(value)).toEqual([HASH_A, HASH_B]);
});
it('skips long strings that do not start with the blob scheme when maxStringLength is set', () => {
const value = `${'x'.repeat(200)} ${uri(HASH_A)}`;
expect(extractBlobHashesFromString(value, 100)).toEqual([]);
});
it('still scans long strings that start with the blob scheme', () => {
const value = `${uri(HASH_A)}${'y'.repeat(200)}`;
expect(extractBlobHashesFromString(value, 100)).toEqual([HASH_A]);
});
});
describe('extractBlobHashesFromValue', () => {
it('extracts from a string', () => {
expect(extractBlobHashesFromValue(uri(HASH_A))).toEqual([HASH_A]);
});
it('extracts from an object with a valid hash field', () => {
expect(extractBlobHashesFromValue({ hash: HASH_A.toUpperCase() })).toEqual([HASH_A]);
});
it('extracts from an object with a uri field', () => {
expect(extractBlobHashesFromValue({ uri: uri(HASH_B) })).toEqual([HASH_B]);
});
it('returns an empty array for non-blob primitives and objects', () => {
expect(extractBlobHashesFromValue(42)).toEqual([]);
expect(extractBlobHashesFromValue(null)).toEqual([]);
expect(extractBlobHashesFromValue({ hash: 'not-a-hash' })).toEqual([]);
});
});
describe('collectBlobHashes', () => {
it('collects and deduplicates hashes across a nested structure', () => {
const value = {
results: [{ response: { output: uri(HASH_A) } }, { response: { output: uri(HASH_A) } }],
traces: [{ metadata: { attachment: uri(HASH_B) } }],
};
expect([...collectBlobHashes(value)].sort()).toEqual([HASH_A, HASH_B]);
});
it('terminates on cyclic objects without infinite recursion', () => {
const node: Record<string, unknown> = { output: uri(HASH_A) };
node.self = node;
expect([...collectBlobHashes(node)]).toEqual([HASH_A]);
});
it('stops recursing past maxDepth', () => {
// Root object is depth 0, the `b` object depth 1, its blob URI string depth 2.
const value = { a: { b: uri(HASH_A) } };
expect([...collectBlobHashes(value, { maxDepth: 1 })]).toEqual([]);
expect([...collectBlobHashes(value, { maxDepth: 2 })]).toEqual([HASH_A]);
});
it('does not overflow the stack on deeply nested input', () => {
let node: unknown = uri(HASH_A);
for (let depth = 0; depth < 50_000; depth += 1) {
node = { child: node };
}
// maxDepth bounds recursion long before the call stack is exhausted.
expect(() => collectBlobHashes(node, { maxDepth: 64 })).not.toThrow();
});
it('honors maxStringLength while still scanning blob-scheme-prefixed strings', () => {
const buried = { output: `${'x'.repeat(200)} ${uri(HASH_A)}` };
const prefixed = { output: `${uri(HASH_B)}${'y'.repeat(200)}` };
expect([...collectBlobHashes(buried, { maxStringLength: 100 })]).toEqual([]);
expect([...collectBlobHashes(prefixed, { maxStringLength: 100 })]).toEqual([HASH_B]);
});
});
+563
View File
@@ -0,0 +1,563 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
extractAndStoreBinaryData,
isBlobStorageEnabled,
normalizeAudioMimeType,
} from '../../src/blobs/extractor';
import { sha256 } from '../../src/util/createHash';
import type { ProviderResponse } from '../../src/types/providers';
// Mock the remoteUpload module
vi.mock('../../src/blobs/remoteUpload', () => ({
uploadBlobRemote: vi.fn(),
}));
// Mock the blob index module
vi.mock('../../src/blobs/index', () => ({
storeBlob: vi.fn().mockResolvedValue({
ref: {
uri: 'promptfoo://blob/abc123',
hash: 'abc123',
},
}),
recordBlobReference: vi.fn().mockResolvedValue(undefined),
}));
describe('normalizeAudioMimeType', () => {
describe('undefined and empty inputs', () => {
it('should return audio/wav for undefined format', () => {
expect(normalizeAudioMimeType(undefined)).toBe('audio/wav');
});
it('should return audio/wav for empty string', () => {
expect(normalizeAudioMimeType('')).toBe('audio/wav');
});
it('should return audio/wav for whitespace-only string', () => {
expect(normalizeAudioMimeType(' ')).toBe('audio/wav');
});
});
describe('already valid MIME types', () => {
it('should pass through valid audio MIME types unchanged', () => {
expect(normalizeAudioMimeType('audio/wav')).toBe('audio/wav');
expect(normalizeAudioMimeType('audio/mpeg')).toBe('audio/mpeg');
expect(normalizeAudioMimeType('audio/ogg')).toBe('audio/ogg');
expect(normalizeAudioMimeType('audio/flac')).toBe('audio/flac');
expect(normalizeAudioMimeType('audio/aac')).toBe('audio/aac');
expect(normalizeAudioMimeType('audio/mp4')).toBe('audio/mp4');
expect(normalizeAudioMimeType('audio/webm')).toBe('audio/webm');
});
it('should pass through vendor MIME types without periods', () => {
expect(normalizeAudioMimeType('audio/x-custom')).toBe('audio/x-custom');
expect(normalizeAudioMimeType('audio/x-wav')).toBe('audio/x-wav');
});
it('should reject MIME types with periods to prevent injection attacks', () => {
// Periods could enable attacks like "audio/wav.html" being interpreted as HTML
expect(normalizeAudioMimeType('audio/vnd.company.format')).toBe('audio/wav');
expect(normalizeAudioMimeType('audio/wav.html')).toBe('audio/wav');
});
});
describe('short format normalization', () => {
it('should normalize common short formats to MIME types', () => {
expect(normalizeAudioMimeType('wav')).toBe('audio/wav');
expect(normalizeAudioMimeType('mp3')).toBe('audio/mpeg');
expect(normalizeAudioMimeType('ogg')).toBe('audio/ogg');
expect(normalizeAudioMimeType('flac')).toBe('audio/flac');
expect(normalizeAudioMimeType('aac')).toBe('audio/aac');
expect(normalizeAudioMimeType('m4a')).toBe('audio/mp4');
expect(normalizeAudioMimeType('webm')).toBe('audio/webm');
});
it('should be case-insensitive for known formats', () => {
expect(normalizeAudioMimeType('WAV')).toBe('audio/wav');
expect(normalizeAudioMimeType('Mp3')).toBe('audio/mpeg');
expect(normalizeAudioMimeType('OGG')).toBe('audio/ogg');
expect(normalizeAudioMimeType('FLAC')).toBe('audio/flac');
});
it('should handle whitespace around format', () => {
expect(normalizeAudioMimeType(' wav ')).toBe('audio/wav');
expect(normalizeAudioMimeType('\tmp3\n')).toBe('audio/mpeg');
});
});
describe('unknown formats', () => {
it('should prefix unknown alphanumeric formats with audio/', () => {
expect(normalizeAudioMimeType('opus')).toBe('audio/opus');
expect(normalizeAudioMimeType('pcm')).toBe('audio/pcm');
expect(normalizeAudioMimeType('raw')).toBe('audio/raw');
});
it('should allow dash and underscore in unknown formats', () => {
expect(normalizeAudioMimeType('x-custom')).toBe('audio/x-custom');
expect(normalizeAudioMimeType('codec_v2')).toBe('audio/codec_v2');
expect(normalizeAudioMimeType('x-wav-hd')).toBe('audio/x-wav-hd');
});
it('should reject unknown formats with periods to prevent MIME injection', () => {
// Periods could enable attacks like "wav.html" -> "audio/wav.html"
expect(normalizeAudioMimeType('vnd.company.format')).toBe('audio/wav');
expect(normalizeAudioMimeType('format-1.0')).toBe('audio/wav');
expect(normalizeAudioMimeType('wav.html')).toBe('audio/wav');
});
});
describe('security: invalid formats', () => {
it('should reject formats with path traversal attempts', () => {
expect(normalizeAudioMimeType('../../etc/passwd')).toBe('audio/wav');
expect(normalizeAudioMimeType('../../../etc/shadow')).toBe('audio/wav');
expect(normalizeAudioMimeType('path/to/file')).toBe('audio/wav');
});
it('should reject formats with newline injection', () => {
expect(normalizeAudioMimeType('audio\nwav')).toBe('audio/wav');
expect(normalizeAudioMimeType('wav\r\nContent-Type: text/html')).toBe('audio/wav');
});
it('should reject formats with header injection characters', () => {
expect(normalizeAudioMimeType('audio;wav')).toBe('audio/wav');
expect(normalizeAudioMimeType('audio:wav')).toBe('audio/wav');
expect(normalizeAudioMimeType('audio wav')).toBe('audio/wav');
});
it('should reject formats with backslashes', () => {
expect(normalizeAudioMimeType('audio\\wav')).toBe('audio/wav');
expect(normalizeAudioMimeType('..\\..\\etc\\passwd')).toBe('audio/wav');
});
it('should reject formats with multiple path components', () => {
expect(normalizeAudioMimeType('audio/wav/extra')).toBe('audio/wav');
});
});
});
describe('Audio MIME type normalization (integration)', () => {
// Skip tests if blob storage is disabled
const maybeIt = isBlobStorageEnabled() ? it : it.skip;
describe('extractAndStoreBinaryData with audio', () => {
maybeIt('should process response with top-level audio', async () => {
const response: ProviderResponse = {
output: 'test',
audio: {
data: 'SGVsbG8gV29ybGQ=',
format: 'wav',
},
};
const result = await extractAndStoreBinaryData(response);
expect(result).toBeDefined();
expect(result?.audio).toBeDefined();
});
maybeIt('should process response with turns audio', async () => {
const response: ProviderResponse = {
output: 'test',
turns: [
{
audio: {
data: 'SGVsbG8gV29ybGQ=',
format: 'mp3',
},
},
],
} as any;
const result = await extractAndStoreBinaryData(response);
expect(result).toBeDefined();
expect((result as any)?.turns).toBeDefined();
});
maybeIt('should handle response without audio', async () => {
const response: ProviderResponse = {
output: 'test',
};
const result = await extractAndStoreBinaryData(response);
expect(result).toEqual(response);
});
maybeIt('should handle null response', async () => {
const result = await extractAndStoreBinaryData(null);
expect(result).toBeNull();
});
maybeIt('should handle undefined response', async () => {
const result = await extractAndStoreBinaryData(undefined);
expect(result).toBeUndefined();
});
});
});
describe('Local blob extraction', () => {
let mockUploadBlobRemote: ReturnType<typeof vi.fn>;
let mockStoreBlob: ReturnType<typeof vi.fn>;
beforeEach(async () => {
vi.resetAllMocks();
// Get the mocked functions
const remoteUploadModule = await import('../../src/blobs/remoteUpload');
mockUploadBlobRemote = vi.mocked(remoteUploadModule.uploadBlobRemote);
const blobIndexModule = await import('../../src/blobs/index');
mockStoreBlob = vi.mocked(blobIndexModule.storeBlob);
// Default mock implementations
mockStoreBlob.mockResolvedValue({
ref: {
uri: 'promptfoo://blob/abc123def456',
hash: 'abc123def456',
},
});
});
afterEach(() => {
vi.resetAllMocks();
});
it('stores extracted blobs locally without eagerly uploading them', async () => {
// Create a large enough data URL to trigger externalization (>1KB)
const largeBase64 = Buffer.alloc(2000).toString('base64');
const response: ProviderResponse = {
output: `data:image/png;base64,${largeBase64}`,
};
await extractAndStoreBinaryData(response);
// Should store locally
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockUploadBlobRemote).not.toHaveBeenCalled();
});
it('should externalize image data URIs to blobRefs', async () => {
const largeBase64 = Buffer.alloc(2000).toString('base64');
const response: ProviderResponse = {
output: 'text output',
images: [{ data: `data:image/png;base64,${largeBase64}`, mimeType: 'image/png' }],
};
const result = await extractAndStoreBinaryData(response);
expect(result?.images?.[0].data).toBeUndefined();
expect(result?.images?.[0].blobRef).toBeDefined();
expect(result?.images?.[0].blobRef?.uri).toContain('promptfoo://blob/');
});
it('should reuse the same image blob when output and images contain identical data URIs', async () => {
const largeBase64 = Buffer.alloc(2000).toString('base64');
const dataUri = `data:image/png;base64,${largeBase64}`;
const response: ProviderResponse = {
output: dataUri,
images: [{ data: dataUri, mimeType: 'image/png' }],
};
const result = await extractAndStoreBinaryData(response);
expect(result?.output).toBe(result?.images?.[0].blobRef?.uri);
expect(result?.images?.[0].data).toBeUndefined();
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockStoreBlob).toHaveBeenCalledWith(
expect.any(Buffer),
'image/png',
expect.objectContaining({
location: 'response.images[0].data',
kind: 'image',
}),
);
});
it('should reuse the same image blob for concurrent identical images[] siblings', async () => {
const largeBase64 = Buffer.alloc(2000).toString('base64');
const dataUri = `data:image/png;base64,${largeBase64}`;
const response: ProviderResponse = {
output: 'text output',
images: [
{ data: dataUri, mimeType: 'image/png' },
{ data: dataUri, mimeType: 'image/png' },
],
};
const result = await extractAndStoreBinaryData(response);
expect(result?.images?.[0].data).toBeUndefined();
expect(result?.images?.[1].data).toBeUndefined();
expect(result?.images?.[0].blobRef).toEqual(result?.images?.[1].blobRef);
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
});
it('should pass through images without data URIs unchanged', async () => {
const response: ProviderResponse = {
output: 'text output',
images: [
{
blobRef: {
uri: 'promptfoo://blob/existing',
hash: 'existing',
mimeType: 'image/png',
sizeBytes: 100,
provider: 'local',
},
},
],
};
const result = await extractAndStoreBinaryData(response);
expect(result).toEqual(response);
});
it('should handle mixed images: externalize data URIs but keep existing blobRefs', async () => {
const largeBase64 = Buffer.alloc(2000).toString('base64');
const response: ProviderResponse = {
output: 'text output',
images: [
{ data: `data:image/png;base64,${largeBase64}`, mimeType: 'image/png' },
{
blobRef: {
uri: 'promptfoo://blob/existing',
hash: 'existing',
mimeType: 'image/jpeg',
sizeBytes: 100,
provider: 'local',
},
},
],
};
const result = await extractAndStoreBinaryData(response);
// First image externalized
expect(result?.images?.[0].data).toBeUndefined();
expect(result?.images?.[0].blobRef).toBeDefined();
// Second image kept as-is
expect(result?.images?.[1].blobRef?.hash).toBe('existing');
});
it('should record blob references embedded in output text', async () => {
const blobIndexModule = await import('../../src/blobs/index');
const mockRecordBlobReference = vi.mocked(blobIndexModule.recordBlobReference);
const firstHash = 'a'.repeat(64);
const secondHash = 'b'.repeat(64);
const response: ProviderResponse = {
output:
`First promptfoo://blob/${firstHash} repeated promptfoo://blob/${firstHash} ` +
`second promptfoo://blob/${secondHash}`,
};
await extractAndStoreBinaryData(response, {
evalId: 'eval-mixed-output',
testIdx: 1,
promptIdx: 2,
});
expect(mockRecordBlobReference).toHaveBeenCalledTimes(2);
expect(mockRecordBlobReference).toHaveBeenCalledWith(
firstHash,
expect.objectContaining({
evalId: 'eval-mixed-output',
testIdx: 1,
promptIdx: 2,
location: 'response.output',
}),
);
expect(mockRecordBlobReference).toHaveBeenCalledWith(
secondHash,
expect.objectContaining({
evalId: 'eval-mixed-output',
testIdx: 1,
promptIdx: 2,
location: 'response.output',
}),
);
});
it('should reuse the top-level audio blob for mirrored realtime metadata audio data', async () => {
const largeBase64 = Buffer.alloc(2000).toString('base64');
const response: ProviderResponse = {
output: 'test',
audio: {
data: largeBase64,
format: 'wav',
transcript: 'hello',
},
metadata: {
audio: {
data: largeBase64,
format: 'wav',
transcript: 'hello',
},
},
};
const result = await extractAndStoreBinaryData(response);
expect(result?.audio?.data).toBeUndefined();
expect(result?.audio?.blobRef).toBeDefined();
expect(result?.metadata?.audio?.data).toBeUndefined();
expect(result?.metadata?.audio?.blobRef).toEqual(result?.audio?.blobRef);
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockStoreBlob).toHaveBeenCalledWith(
expect.any(Buffer),
'audio/wav',
expect.objectContaining({
location: 'response.audio.data',
kind: 'audio',
}),
);
});
it('should externalize metadata-only audio data', async () => {
const largeBase64 = Buffer.alloc(2000).toString('base64');
const response: ProviderResponse = {
output: 'test',
metadata: {
audio: {
data: largeBase64,
format: 'wav',
},
},
};
const result = await extractAndStoreBinaryData(response);
expect(result?.metadata?.audio?.data).toBeUndefined();
expect(result?.metadata?.audio?.blobRef).toBeDefined();
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockStoreBlob).toHaveBeenCalledWith(
expect.any(Buffer),
'audio/wav',
expect.objectContaining({
location: 'response.metadata.audio.data',
kind: 'audio',
}),
);
});
it('should reuse the same audio blob across turns[].audio and metadata.audio', async () => {
// Realtime providers can mirror an audio chunk into both turns[N].audio
// and metadata.audio. Without a shared cache, the metadata path stored a
// separate blob.
const largeBase64 = Buffer.alloc(2000).toString('base64');
const response = {
output: 'test',
turns: [
{
role: 'assistant',
audio: { data: largeBase64, format: 'wav', transcript: 'hi' },
},
],
metadata: {
audio: { data: largeBase64, format: 'wav', transcript: 'hi' },
},
} as unknown as ProviderResponse;
const result = await extractAndStoreBinaryData(response);
const turnsBlob = (result as any)?.turns?.[0]?.audio?.blobRef;
expect(turnsBlob).toBeDefined();
expect(result?.metadata?.audio?.blobRef).toEqual(turnsBlob);
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockStoreBlob).toHaveBeenCalledWith(
expect.any(Buffer),
'audio/wav',
expect.objectContaining({
location: 'response.turns[0].audio.data',
kind: 'audio',
}),
);
});
it('should reuse the same image blob across images[] and a metadata data-URL of the same bytes', async () => {
// E.g. a provider that mirrors a generated image into both `images[]` and
// a metadata field. The shared per-response cache canonicalizes on the
// parsed bytes so both paths land on one blob.
const largeBase64 = Buffer.alloc(2000).toString('base64');
const dataUri = `data:image/png;base64,${largeBase64}`;
const response: ProviderResponse = {
output: 'text',
images: [{ data: dataUri, mimeType: 'image/png' }],
metadata: {
previewImage: dataUri,
},
};
const result = await extractAndStoreBinaryData(response);
expect(result?.images?.[0].blobRef?.uri).toBeDefined();
expect(result?.metadata?.previewImage).toBe(result?.images?.[0].blobRef?.uri);
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockStoreBlob).toHaveBeenCalledWith(
expect.any(Buffer),
'image/png',
expect.objectContaining({
location: 'response.images[0].data',
kind: 'image',
}),
);
});
it('should reuse the same image blob when output is a data URL and metadata holds raw base64', async () => {
// Different encodings of identical bytes (data: URL vs. raw base64) must
// hit the same cache slot — canonicalization happens on the parsed buffer.
const largeBase64 = Buffer.alloc(2000).toString('base64');
const dataUri = `data:image/png;base64,${largeBase64}`;
const response: ProviderResponse = {
output: dataUri,
images: [{ data: `data:image/png;base64,${largeBase64}`, mimeType: 'image/png' }],
};
const result = await extractAndStoreBinaryData(response);
expect(result?.output).toBe(result?.images?.[0].blobRef?.uri);
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
});
it('should store small raw SVG outputs for the media library without replacing text output', async () => {
const svg = '<svg xmlns="http://www.w3.org/2000/svg"><circle r="4"/></svg>';
const response: ProviderResponse = { output: svg };
const result = await extractAndStoreBinaryData(response);
expect(result?.output).toBe(svg);
expect(result?.metadata?.blobUris).toEqual(['promptfoo://blob/abc123def456']);
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
expect(mockStoreBlob).toHaveBeenCalledWith(
Buffer.from(svg),
'image/svg+xml',
expect.objectContaining({
location: 'response.output',
kind: 'image',
}),
);
});
it('should not store the same raw SVG preview twice when metadata already references it', async () => {
const svg = '<svg xmlns="http://www.w3.org/2000/svg"><circle r="4"/></svg>';
const response: ProviderResponse = {
output: svg,
metadata: {
blobUris: [`promptfoo://blob/${sha256(Buffer.from(svg))}`],
},
};
const result = await extractAndStoreBinaryData(response);
expect(result).toBe(response);
expect(mockStoreBlob).not.toHaveBeenCalled();
});
it('should not store multiple root SVG documents as one media blob', async () => {
const response: ProviderResponse = {
output: '<svg><circle/></svg>\n\n<svg><rect/></svg>',
};
const result = await extractAndStoreBinaryData(response);
expect(result).toBe(response);
expect(mockStoreBlob).not.toHaveBeenCalled();
});
});
+43
View File
@@ -0,0 +1,43 @@
import { afterEach, describe, expect, it } from 'vitest';
import { FilesystemBlobStorageProvider } from '../../src/blobs/filesystemProvider';
import { createTempDir, removeTempDir } from '../util/utils';
describe('FilesystemBlobStorageProvider', () => {
let tempDir: string | undefined;
afterEach(() => {
removeTempDir(tempDir);
tempDir = undefined;
});
it('stores and retrieves blobs by hash', async () => {
tempDir = createTempDir('promptfoo-blobs-');
const provider = new FilesystemBlobStorageProvider({ basePath: tempDir });
const data = Buffer.from('blob-data');
const { ref } = await provider.store(data, 'application/octet-stream');
const stored = await provider.getByHash(ref.hash);
expect(stored.data.toString('utf8')).toBe('blob-data');
});
it('deduplicates identical blobs', async () => {
tempDir = createTempDir('promptfoo-blobs-');
const provider = new FilesystemBlobStorageProvider({ basePath: tempDir });
const data = Buffer.from('same');
const first = await provider.store(data, 'application/octet-stream');
const second = await provider.store(data, 'application/octet-stream');
expect(first.ref.hash).toBe(second.ref.hash);
expect(second.deduplicated).toBe(true);
});
it('rejects invalid hashes and prevents path traversal', async () => {
tempDir = createTempDir('promptfoo-blobs-');
const provider = new FilesystemBlobStorageProvider({ basePath: tempDir });
await expect(provider.exists('../../etc/passwd')).resolves.toBe(false);
await expect(provider.getByHash('../../etc/passwd')).rejects.toThrow(/invalid blob hash/i);
});
});
+217
View File
@@ -0,0 +1,217 @@
import { randomUUID } from 'node:crypto';
import { eq, inArray } from 'drizzle-orm';
import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import {
getShareAuthorizedBlob,
isBlobAllowedForShare,
recordBlobReference,
resetBlobStorageProvider,
setBlobStorageProvider,
} from '../../src/blobs';
import { getDb } from '../../src/database';
import { blobAssetsTable, blobReferencesTable, evalsTable } from '../../src/database/tables';
import { runDbMigrations } from '../../src/migrate';
import type { BlobStorageProvider } from '../../src/blobs';
describe('blob share authorization', () => {
const evalId = `eval-${randomUUID()}`;
const otherEvalId = `eval-${randomUUID()}`;
const trustedHash = 'a'.repeat(64);
const unclassifiedHash = 'b'.repeat(64);
const importedHash = 'c'.repeat(64);
const otherEvalHash = 'd'.repeat(64);
const hashes = [trustedHash, unclassifiedHash, importedHash, otherEvalHash];
beforeAll(async () => {
await runDbMigrations();
});
beforeEach(async () => {
const db = await getDb();
await db.insert(evalsTable).values([
{ id: evalId, config: {}, results: {} },
{ id: otherEvalId, config: {}, results: {} },
]);
await db.insert(blobAssetsTable).values(
hashes.map((hash) => ({
hash,
mimeType: 'image/png',
provider: 'filesystem',
sizeBytes: 1,
})),
);
await db.insert(blobReferencesTable).values([
{
id: randomUUID(),
blobHash: trustedHash,
evalId,
kind: 'image',
location: 'response.output',
},
{
id: randomUUID(),
blobHash: unclassifiedHash,
evalId,
location: 'response.output',
},
{
id: randomUUID(),
blobHash: importedHash,
evalId,
location: 'import',
},
{
id: randomUUID(),
blobHash: otherEvalHash,
evalId: otherEvalId,
kind: 'image',
location: 'response.output',
},
]);
});
afterEach(async () => {
const db = await getDb();
await db
.delete(blobReferencesTable)
.where(inArray(blobReferencesTable.evalId, [evalId, otherEvalId]));
await db.delete(blobAssetsTable).where(inArray(blobAssetsTable.hash, hashes));
await db.delete(evalsTable).where(inArray(evalsTable.id, [evalId, otherEvalId]));
});
it('allows only trusted references associated with the local eval', async () => {
await expect(isBlobAllowedForShare(trustedHash, evalId)).resolves.toBe(true);
await expect(isBlobAllowedForShare(importedHash, evalId)).resolves.toBe(true);
await expect(isBlobAllowedForShare(unclassifiedHash, evalId)).resolves.toBe(false);
await expect(isBlobAllowedForShare(otherEvalHash, evalId)).resolves.toBe(false);
});
it('does not authorize a trusted hash after its eval association is removed', async () => {
const db = await getDb();
await db.delete(blobReferencesTable).where(eq(blobReferencesTable.blobHash, trustedHash));
await expect(isBlobAllowedForShare(trustedHash, evalId)).resolves.toBe(false);
});
it('reads local bytes only for share-authorized references', async () => {
setBlobStorageProvider({
providerId: 'test-stub',
store: async () => {
throw new Error('not implemented');
},
getByHash: async (hash: string) => ({
data: Buffer.from('trusted-bytes'),
metadata: {
createdAt: '2026-06-08T00:00:00.000Z',
key: hash,
mimeType: 'image/png',
provider: 'test-stub',
sizeBytes: 13,
},
}),
exists: async () => true,
deleteByHash: async () => {},
getUrl: async () => null,
});
try {
await expect(getShareAuthorizedBlob(unclassifiedHash, evalId)).resolves.toBeNull();
await expect(getShareAuthorizedBlob(otherEvalHash, evalId)).resolves.toBeNull();
const blob = await getShareAuthorizedBlob(trustedHash, evalId);
expect(blob?.data.toString()).toBe('trusted-bytes');
} finally {
resetBlobStorageProvider();
}
});
});
describe('recordBlobReference provenance upgrades', () => {
const evalId = `eval-${randomUUID()}`;
const hash = 'e'.repeat(64);
const stubProvider: BlobStorageProvider = {
providerId: 'test-stub',
store: async () => {
throw new Error('not implemented');
},
getByHash: async () => {
throw new Error('not implemented');
},
exists: async () => true,
deleteByHash: async () => {},
getUrl: async () => null,
};
beforeAll(async () => {
await runDbMigrations();
});
beforeEach(async () => {
setBlobStorageProvider(stubProvider);
const db = await getDb();
await db.insert(evalsTable).values([{ id: evalId, config: {}, results: {} }]);
await db
.insert(blobAssetsTable)
.values([{ hash, mimeType: 'image/png', provider: 'filesystem', sizeBytes: 1 }]);
});
afterEach(async () => {
resetBlobStorageProvider();
const db = await getDb();
await db.delete(blobReferencesTable).where(eq(blobReferencesTable.evalId, evalId));
await db.delete(blobAssetsTable).where(eq(blobAssetsTable.hash, hash));
await db.delete(evalsTable).where(eq(evalsTable.id, evalId));
});
async function getReferenceRows() {
const db = await getDb();
return db.select().from(blobReferencesTable).where(eq(blobReferencesTable.evalId, evalId));
}
it('upgrades an unclassified reference once the blob is independently classified', async () => {
await recordBlobReference(hash, { evalId, location: 'response.output' });
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(false);
await recordBlobReference(hash, { evalId, kind: 'image', location: 'response.images[0].data' });
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(true);
const rows = await getReferenceRows();
expect(rows).toHaveLength(1);
expect(rows[0].kind).toBe('image');
});
it('does not authorize re-recorded scan references that carry no kind', async () => {
await recordBlobReference(hash, { evalId, location: 'response.output' });
await recordBlobReference(hash, { evalId, location: 'response.metadata' });
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(false);
const rows = await getReferenceRows();
expect(rows).toHaveLength(1);
expect(rows[0].kind).toBeNull();
});
it('does not downgrade a classified reference when re-recorded without a kind', async () => {
await recordBlobReference(hash, { evalId, kind: 'image', location: 'response.output' });
await recordBlobReference(hash, { evalId, location: 'response.metadata' });
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(true);
const rows = await getReferenceRows();
expect(rows).toHaveLength(1);
expect(rows[0].kind).toBe('image');
});
it('upgrades a scan reference to import provenance', async () => {
await recordBlobReference(hash, { evalId, location: 'response.output' });
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(false);
await recordBlobReference(hash, { evalId, location: 'import' });
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(true);
const rows = await getReferenceRows();
expect(rows).toHaveLength(1);
expect(rows[0].location).toBe('import');
});
});
+98
View File
@@ -0,0 +1,98 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { shouldAttemptRemoteBlobUpload, uploadBlobRemote } from '../../src/blobs/remoteUpload';
import { getEnvBool } from '../../src/envars';
import { isLoggedIntoCloud } from '../../src/globalConfig/accounts';
import { cloudConfig } from '../../src/globalConfig/cloud';
import { fetchWithProxy } from '../../src/util/fetch/index';
vi.mock('../../src/envars', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../src/envars')>();
return {
...actual,
getEnvBool: vi.fn(),
};
});
vi.mock('../../src/globalConfig/accounts', () => ({
isLoggedIntoCloud: vi.fn(),
}));
vi.mock('../../src/globalConfig/cloud', () => ({
cloudConfig: {
getApiHost: vi.fn(),
getApiKey: vi.fn(),
},
}));
vi.mock('../../src/util/fetch/index', () => ({
fetchWithProxy: vi.fn(),
}));
describe('remote blob upload', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(getEnvBool).mockReturnValue(false);
vi.mocked(isLoggedIntoCloud).mockReturnValue(true);
vi.mocked(cloudConfig.getApiHost).mockReturnValue('https://api.example.com');
vi.mocked(cloudConfig.getApiKey).mockReturnValue('test-api-key');
vi.mocked(fetchWithProxy).mockResolvedValue({
ok: true,
status: 200,
statusText: 'OK',
json: async () => ({
ref: {
uri: 'promptfoo://blob/abc123',
hash: 'abc123',
},
}),
} as Response);
});
afterEach(() => {
vi.resetAllMocks();
});
it('attempts remote upload when sharing is enabled and Cloud auth is configured', () => {
expect(shouldAttemptRemoteBlobUpload()).toBe(true);
expect(cloudConfig.getApiHost).toHaveBeenCalledTimes(1);
expect(cloudConfig.getApiKey).toHaveBeenCalledTimes(1);
});
it('does not attempt remote upload when PROMPTFOO_DISABLE_SHARING is set', async () => {
vi.mocked(getEnvBool).mockImplementation((key) => key === 'PROMPTFOO_DISABLE_SHARING');
expect(shouldAttemptRemoteBlobUpload()).toBe(false);
const result = await uploadBlobRemote(Buffer.from('image-bytes'), 'image/png', {
evalId: 'eval-123',
location: 'response.output',
kind: 'image',
});
expect(result).toBeNull();
expect(cloudConfig.getApiHost).not.toHaveBeenCalled();
expect(cloudConfig.getApiKey).not.toHaveBeenCalled();
expect(fetchWithProxy).not.toHaveBeenCalled();
});
it('posts blobs to the Cloud blob endpoint when allowed', async () => {
await uploadBlobRemote(Buffer.from('image-bytes'), 'image/png', {
evalId: 'eval-123',
location: 'response.output',
kind: 'image',
});
expect(fetchWithProxy).toHaveBeenCalledWith(
'https://api.example.com/api/blobs',
expect.objectContaining({
method: 'POST',
headers: expect.objectContaining({
Authorization: 'Bearer test-api-key',
'Content-Type': 'application/json',
}),
}),
);
expect(cloudConfig.getApiHost).toHaveBeenCalledTimes(1);
expect(cloudConfig.getApiKey).toHaveBeenCalledTimes(1);
});
});
+265
View File
@@ -0,0 +1,265 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { getShareAuthorizedBlob } from '../../src/blobs';
import { uploadBlobRemote } from '../../src/blobs/remoteUpload';
import { createRemoteBlobUploadCache, uploadBlobRefsForShare } from '../../src/blobs/shareUpload';
import logger from '../../src/logger';
import type { StoredBlob } from '../../src/blobs';
vi.mock('../../src/blobs', () => ({
getShareAuthorizedBlob: vi.fn(),
}));
vi.mock('../../src/blobs/remoteUpload', () => ({
uploadBlobRemote: vi.fn(),
}));
vi.mock('../../src/logger', () => ({
default: {
warn: vi.fn(),
},
}));
const storedBlob: StoredBlob = {
data: Buffer.from('image-bytes'),
metadata: {
createdAt: '2026-06-08T00:00:00.000Z',
key: 'blob-key',
mimeType: 'image/png',
provider: 'filesystem',
sizeBytes: 11,
},
};
describe('share-time blob upload', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(getShareAuthorizedBlob).mockResolvedValue(storedBlob);
vi.mocked(uploadBlobRemote).mockResolvedValue({
deduplicated: false,
ref: {
hash: 'a'.repeat(64),
mimeType: 'image/png',
provider: 'cloud',
sizeBytes: 11,
uri: `promptfoo://blob/${'a'.repeat(64)}`,
},
});
});
it('uploads each referenced blob only once when the share context is unchanged', async () => {
const hash = 'a'.repeat(64);
const cache = createRemoteBlobUploadCache();
await uploadBlobRefsForShare(
{
output: `promptfoo://blob/${hash}`,
images: [{ blobRef: { hash, uri: `promptfoo://blob/${hash}` } }],
},
cache,
{ localEvalId: 'local-eval-123', remoteEvalId: 'remote-eval-123', promptIdx: 2, testIdx: 1 },
);
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
localEvalId: 'local-eval-123',
remoteEvalId: 'remote-eval-123',
promptIdx: 2,
testIdx: 1,
});
expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
expect(getShareAuthorizedBlob).toHaveBeenCalledWith(hash, 'local-eval-123');
expect(uploadBlobRemote).toHaveBeenCalledOnce();
expect(uploadBlobRemote).toHaveBeenCalledWith(Buffer.from('image-bytes'), 'image/png', {
evalId: 'remote-eval-123',
kind: 'image',
location: 'share',
promptIdx: 2,
testIdx: 1,
});
});
it('uploads the same blob again when it appears in a different result row', async () => {
const hash = '7'.repeat(64);
const cache = createRemoteBlobUploadCache();
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
localEvalId: 'local-eval-123',
remoteEvalId: 'remote-eval-123',
promptIdx: 2,
testIdx: 1,
});
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
localEvalId: 'local-eval-123',
remoteEvalId: 'remote-eval-123',
promptIdx: 4,
testIdx: 3,
});
expect(getShareAuthorizedBlob).toHaveBeenCalledTimes(2);
expect(getShareAuthorizedBlob).toHaveBeenNthCalledWith(1, hash, 'local-eval-123');
expect(getShareAuthorizedBlob).toHaveBeenNthCalledWith(2, hash, 'local-eval-123');
expect(uploadBlobRemote).toHaveBeenCalledTimes(2);
expect(uploadBlobRemote).toHaveBeenNthCalledWith(1, Buffer.from('image-bytes'), 'image/png', {
evalId: 'remote-eval-123',
kind: 'image',
location: 'share',
promptIdx: 2,
testIdx: 1,
});
expect(uploadBlobRemote).toHaveBeenNthCalledWith(2, Buffer.from('image-bytes'), 'image/png', {
evalId: 'remote-eval-123',
kind: 'image',
location: 'share',
promptIdx: 4,
testIdx: 3,
});
});
it('keeps a zero-index result row distinct from a coordinate-less reference', async () => {
// Pins the `?? null` (not `||`) choice in getUploadCacheKey: promptIdx/testIdx
// of 0 is the first row of every eval and must not collapse into a reference
// that carries no coordinates. A regression to `|| null` would dedupe these two
// into a single upload, dropping the (0, 0) row's provenance.
const hash = '0'.repeat(64);
const cache = createRemoteBlobUploadCache();
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
localEvalId: 'local-eval-zero',
remoteEvalId: 'remote-eval-zero',
promptIdx: 0,
testIdx: 0,
});
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
localEvalId: 'local-eval-zero',
remoteEvalId: 'remote-eval-zero',
});
expect(getShareAuthorizedBlob).toHaveBeenCalledTimes(2);
expect(uploadBlobRemote).toHaveBeenCalledTimes(2);
expect(uploadBlobRemote).toHaveBeenNthCalledWith(1, Buffer.from('image-bytes'), 'image/png', {
evalId: 'remote-eval-zero',
kind: 'image',
location: 'share',
promptIdx: 0,
testIdx: 0,
});
expect(uploadBlobRemote).toHaveBeenNthCalledWith(2, Buffer.from('image-bytes'), 'image/png', {
evalId: 'remote-eval-zero',
kind: 'image',
location: 'share',
promptIdx: undefined,
testIdx: undefined,
});
});
it('does not upload a copied blob URI that is not share-authorized for the eval', async () => {
const hash = 'd'.repeat(64);
vi.mocked(getShareAuthorizedBlob).mockResolvedValue(null);
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
localEvalId: 'local-eval-unauthorized',
remoteEvalId: 'remote-eval-unauthorized',
});
expect(getShareAuthorizedBlob).toHaveBeenCalledWith(hash, 'local-eval-unauthorized');
expect(uploadBlobRemote).not.toHaveBeenCalled();
});
it('logs upload failures without failing the share', async () => {
const hash = 'b'.repeat(64);
vi.mocked(uploadBlobRemote).mockRejectedValue(new Error('network unavailable'));
await expect(
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
localEvalId: 'local-eval-456',
remoteEvalId: 'remote-eval-456',
}),
).resolves.toBeUndefined();
expect(logger.warn).toHaveBeenCalledWith(
'[Share] Failed to upload referenced blob; shared media may be unavailable',
expect.objectContaining({
error: 'network unavailable',
evalId: 'remote-eval-456',
hash,
}),
);
});
it('logs unavailable remote storage without failing the share', async () => {
const hash = 'c'.repeat(64);
vi.mocked(uploadBlobRemote).mockResolvedValue(null);
await expect(
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
localEvalId: 'local-eval-789',
remoteEvalId: 'remote-eval-789',
}),
).resolves.toBeUndefined();
expect(logger.warn).toHaveBeenCalledWith(
'[Share] Failed to upload referenced blob; shared media may be unavailable',
{
evalId: 'remote-eval-789',
hash,
},
);
});
it('checks authorization only once for repeated unauthorized references', async () => {
const hash = 'e'.repeat(64);
vi.mocked(getShareAuthorizedBlob).mockResolvedValue(null);
const cache = createRemoteBlobUploadCache();
const context = { localEvalId: 'local-eval-999', remoteEvalId: 'remote-eval-999' };
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, context);
await uploadBlobRefsForShare({ output: `promptfoo://blob/${hash}` }, cache, context);
expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
expect(uploadBlobRemote).not.toHaveBeenCalled();
});
it('shares one upload across concurrent references to the same blob', async () => {
const hash = 'f'.repeat(64);
let releaseAuth: ((blob: StoredBlob | null) => void) | undefined;
vi.mocked(getShareAuthorizedBlob).mockImplementation(
() =>
new Promise((resolve) => {
releaseAuth = resolve;
}),
);
const cache = createRemoteBlobUploadCache();
const context = { localEvalId: 'local-eval-111', remoteEvalId: 'remote-eval-111' };
const uploads = Promise.all([
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, context),
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, context),
]);
releaseAuth?.(storedBlob);
await uploads;
expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
expect(uploadBlobRemote).toHaveBeenCalledOnce();
});
it('skips the blob without failing the share when the authorization check errors', async () => {
const hash = '9'.repeat(64);
vi.mocked(getShareAuthorizedBlob).mockRejectedValue(new Error('db locked'));
await expect(
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
localEvalId: 'local-eval-222',
remoteEvalId: 'remote-eval-222',
}),
).resolves.toBeUndefined();
expect(uploadBlobRemote).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalledWith(
'[Share] Failed to upload referenced blob; shared media may be unavailable',
expect.objectContaining({
error: 'db locked',
hash,
}),
);
});
});