chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
collectBlobHashes,
|
||||
extractBlobHashesFromString,
|
||||
extractBlobHashesFromValue,
|
||||
normalizeBlobHash,
|
||||
} from '../../src/blobs/blobRefs';
|
||||
|
||||
const HASH_A = 'a'.repeat(64);
|
||||
const HASH_B = 'b'.repeat(64);
|
||||
const uri = (hash: string) => `promptfoo://blob/${hash}`;
|
||||
|
||||
describe('normalizeBlobHash', () => {
|
||||
it('lowercases the hash', () => {
|
||||
expect(normalizeBlobHash('ABCDEF')).toBe('abcdef');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractBlobHashesFromString', () => {
|
||||
it('returns an empty array for strings without a blob URI', () => {
|
||||
expect(extractBlobHashesFromString('just some text')).toEqual([]);
|
||||
});
|
||||
|
||||
it('extracts and normalizes every blob hash in the string', () => {
|
||||
const value = `before ${uri(HASH_A.toUpperCase())} middle ${uri(HASH_B)} after`;
|
||||
expect(extractBlobHashesFromString(value)).toEqual([HASH_A, HASH_B]);
|
||||
});
|
||||
|
||||
it('skips long strings that do not start with the blob scheme when maxStringLength is set', () => {
|
||||
const value = `${'x'.repeat(200)} ${uri(HASH_A)}`;
|
||||
expect(extractBlobHashesFromString(value, 100)).toEqual([]);
|
||||
});
|
||||
|
||||
it('still scans long strings that start with the blob scheme', () => {
|
||||
const value = `${uri(HASH_A)}${'y'.repeat(200)}`;
|
||||
expect(extractBlobHashesFromString(value, 100)).toEqual([HASH_A]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractBlobHashesFromValue', () => {
|
||||
it('extracts from a string', () => {
|
||||
expect(extractBlobHashesFromValue(uri(HASH_A))).toEqual([HASH_A]);
|
||||
});
|
||||
|
||||
it('extracts from an object with a valid hash field', () => {
|
||||
expect(extractBlobHashesFromValue({ hash: HASH_A.toUpperCase() })).toEqual([HASH_A]);
|
||||
});
|
||||
|
||||
it('extracts from an object with a uri field', () => {
|
||||
expect(extractBlobHashesFromValue({ uri: uri(HASH_B) })).toEqual([HASH_B]);
|
||||
});
|
||||
|
||||
it('returns an empty array for non-blob primitives and objects', () => {
|
||||
expect(extractBlobHashesFromValue(42)).toEqual([]);
|
||||
expect(extractBlobHashesFromValue(null)).toEqual([]);
|
||||
expect(extractBlobHashesFromValue({ hash: 'not-a-hash' })).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('collectBlobHashes', () => {
|
||||
it('collects and deduplicates hashes across a nested structure', () => {
|
||||
const value = {
|
||||
results: [{ response: { output: uri(HASH_A) } }, { response: { output: uri(HASH_A) } }],
|
||||
traces: [{ metadata: { attachment: uri(HASH_B) } }],
|
||||
};
|
||||
expect([...collectBlobHashes(value)].sort()).toEqual([HASH_A, HASH_B]);
|
||||
});
|
||||
|
||||
it('terminates on cyclic objects without infinite recursion', () => {
|
||||
const node: Record<string, unknown> = { output: uri(HASH_A) };
|
||||
node.self = node;
|
||||
expect([...collectBlobHashes(node)]).toEqual([HASH_A]);
|
||||
});
|
||||
|
||||
it('stops recursing past maxDepth', () => {
|
||||
// Root object is depth 0, the `b` object depth 1, its blob URI string depth 2.
|
||||
const value = { a: { b: uri(HASH_A) } };
|
||||
expect([...collectBlobHashes(value, { maxDepth: 1 })]).toEqual([]);
|
||||
expect([...collectBlobHashes(value, { maxDepth: 2 })]).toEqual([HASH_A]);
|
||||
});
|
||||
|
||||
it('does not overflow the stack on deeply nested input', () => {
|
||||
let node: unknown = uri(HASH_A);
|
||||
for (let depth = 0; depth < 50_000; depth += 1) {
|
||||
node = { child: node };
|
||||
}
|
||||
// maxDepth bounds recursion long before the call stack is exhausted.
|
||||
expect(() => collectBlobHashes(node, { maxDepth: 64 })).not.toThrow();
|
||||
});
|
||||
|
||||
it('honors maxStringLength while still scanning blob-scheme-prefixed strings', () => {
|
||||
const buried = { output: `${'x'.repeat(200)} ${uri(HASH_A)}` };
|
||||
const prefixed = { output: `${uri(HASH_B)}${'y'.repeat(200)}` };
|
||||
expect([...collectBlobHashes(buried, { maxStringLength: 100 })]).toEqual([]);
|
||||
expect([...collectBlobHashes(prefixed, { maxStringLength: 100 })]).toEqual([HASH_B]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,563 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
extractAndStoreBinaryData,
|
||||
isBlobStorageEnabled,
|
||||
normalizeAudioMimeType,
|
||||
} from '../../src/blobs/extractor';
|
||||
import { sha256 } from '../../src/util/createHash';
|
||||
|
||||
import type { ProviderResponse } from '../../src/types/providers';
|
||||
|
||||
// Mock the remoteUpload module
|
||||
vi.mock('../../src/blobs/remoteUpload', () => ({
|
||||
uploadBlobRemote: vi.fn(),
|
||||
}));
|
||||
|
||||
// Mock the blob index module
|
||||
vi.mock('../../src/blobs/index', () => ({
|
||||
storeBlob: vi.fn().mockResolvedValue({
|
||||
ref: {
|
||||
uri: 'promptfoo://blob/abc123',
|
||||
hash: 'abc123',
|
||||
},
|
||||
}),
|
||||
recordBlobReference: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
describe('normalizeAudioMimeType', () => {
|
||||
describe('undefined and empty inputs', () => {
|
||||
it('should return audio/wav for undefined format', () => {
|
||||
expect(normalizeAudioMimeType(undefined)).toBe('audio/wav');
|
||||
});
|
||||
|
||||
it('should return audio/wav for empty string', () => {
|
||||
expect(normalizeAudioMimeType('')).toBe('audio/wav');
|
||||
});
|
||||
|
||||
it('should return audio/wav for whitespace-only string', () => {
|
||||
expect(normalizeAudioMimeType(' ')).toBe('audio/wav');
|
||||
});
|
||||
});
|
||||
|
||||
describe('already valid MIME types', () => {
|
||||
it('should pass through valid audio MIME types unchanged', () => {
|
||||
expect(normalizeAudioMimeType('audio/wav')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('audio/mpeg')).toBe('audio/mpeg');
|
||||
expect(normalizeAudioMimeType('audio/ogg')).toBe('audio/ogg');
|
||||
expect(normalizeAudioMimeType('audio/flac')).toBe('audio/flac');
|
||||
expect(normalizeAudioMimeType('audio/aac')).toBe('audio/aac');
|
||||
expect(normalizeAudioMimeType('audio/mp4')).toBe('audio/mp4');
|
||||
expect(normalizeAudioMimeType('audio/webm')).toBe('audio/webm');
|
||||
});
|
||||
|
||||
it('should pass through vendor MIME types without periods', () => {
|
||||
expect(normalizeAudioMimeType('audio/x-custom')).toBe('audio/x-custom');
|
||||
expect(normalizeAudioMimeType('audio/x-wav')).toBe('audio/x-wav');
|
||||
});
|
||||
|
||||
it('should reject MIME types with periods to prevent injection attacks', () => {
|
||||
// Periods could enable attacks like "audio/wav.html" being interpreted as HTML
|
||||
expect(normalizeAudioMimeType('audio/vnd.company.format')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('audio/wav.html')).toBe('audio/wav');
|
||||
});
|
||||
});
|
||||
|
||||
describe('short format normalization', () => {
|
||||
it('should normalize common short formats to MIME types', () => {
|
||||
expect(normalizeAudioMimeType('wav')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('mp3')).toBe('audio/mpeg');
|
||||
expect(normalizeAudioMimeType('ogg')).toBe('audio/ogg');
|
||||
expect(normalizeAudioMimeType('flac')).toBe('audio/flac');
|
||||
expect(normalizeAudioMimeType('aac')).toBe('audio/aac');
|
||||
expect(normalizeAudioMimeType('m4a')).toBe('audio/mp4');
|
||||
expect(normalizeAudioMimeType('webm')).toBe('audio/webm');
|
||||
});
|
||||
|
||||
it('should be case-insensitive for known formats', () => {
|
||||
expect(normalizeAudioMimeType('WAV')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('Mp3')).toBe('audio/mpeg');
|
||||
expect(normalizeAudioMimeType('OGG')).toBe('audio/ogg');
|
||||
expect(normalizeAudioMimeType('FLAC')).toBe('audio/flac');
|
||||
});
|
||||
|
||||
it('should handle whitespace around format', () => {
|
||||
expect(normalizeAudioMimeType(' wav ')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('\tmp3\n')).toBe('audio/mpeg');
|
||||
});
|
||||
});
|
||||
|
||||
describe('unknown formats', () => {
|
||||
it('should prefix unknown alphanumeric formats with audio/', () => {
|
||||
expect(normalizeAudioMimeType('opus')).toBe('audio/opus');
|
||||
expect(normalizeAudioMimeType('pcm')).toBe('audio/pcm');
|
||||
expect(normalizeAudioMimeType('raw')).toBe('audio/raw');
|
||||
});
|
||||
|
||||
it('should allow dash and underscore in unknown formats', () => {
|
||||
expect(normalizeAudioMimeType('x-custom')).toBe('audio/x-custom');
|
||||
expect(normalizeAudioMimeType('codec_v2')).toBe('audio/codec_v2');
|
||||
expect(normalizeAudioMimeType('x-wav-hd')).toBe('audio/x-wav-hd');
|
||||
});
|
||||
|
||||
it('should reject unknown formats with periods to prevent MIME injection', () => {
|
||||
// Periods could enable attacks like "wav.html" -> "audio/wav.html"
|
||||
expect(normalizeAudioMimeType('vnd.company.format')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('format-1.0')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('wav.html')).toBe('audio/wav');
|
||||
});
|
||||
});
|
||||
|
||||
describe('security: invalid formats', () => {
|
||||
it('should reject formats with path traversal attempts', () => {
|
||||
expect(normalizeAudioMimeType('../../etc/passwd')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('../../../etc/shadow')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('path/to/file')).toBe('audio/wav');
|
||||
});
|
||||
|
||||
it('should reject formats with newline injection', () => {
|
||||
expect(normalizeAudioMimeType('audio\nwav')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('wav\r\nContent-Type: text/html')).toBe('audio/wav');
|
||||
});
|
||||
|
||||
it('should reject formats with header injection characters', () => {
|
||||
expect(normalizeAudioMimeType('audio;wav')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('audio:wav')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('audio wav')).toBe('audio/wav');
|
||||
});
|
||||
|
||||
it('should reject formats with backslashes', () => {
|
||||
expect(normalizeAudioMimeType('audio\\wav')).toBe('audio/wav');
|
||||
expect(normalizeAudioMimeType('..\\..\\etc\\passwd')).toBe('audio/wav');
|
||||
});
|
||||
|
||||
it('should reject formats with multiple path components', () => {
|
||||
expect(normalizeAudioMimeType('audio/wav/extra')).toBe('audio/wav');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Audio MIME type normalization (integration)', () => {
|
||||
// Skip tests if blob storage is disabled
|
||||
const maybeIt = isBlobStorageEnabled() ? it : it.skip;
|
||||
|
||||
describe('extractAndStoreBinaryData with audio', () => {
|
||||
maybeIt('should process response with top-level audio', async () => {
|
||||
const response: ProviderResponse = {
|
||||
output: 'test',
|
||||
audio: {
|
||||
data: 'SGVsbG8gV29ybGQ=',
|
||||
format: 'wav',
|
||||
},
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.audio).toBeDefined();
|
||||
});
|
||||
|
||||
maybeIt('should process response with turns audio', async () => {
|
||||
const response: ProviderResponse = {
|
||||
output: 'test',
|
||||
turns: [
|
||||
{
|
||||
audio: {
|
||||
data: 'SGVsbG8gV29ybGQ=',
|
||||
format: 'mp3',
|
||||
},
|
||||
},
|
||||
],
|
||||
} as any;
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
expect(result).toBeDefined();
|
||||
expect((result as any)?.turns).toBeDefined();
|
||||
});
|
||||
|
||||
maybeIt('should handle response without audio', async () => {
|
||||
const response: ProviderResponse = {
|
||||
output: 'test',
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
expect(result).toEqual(response);
|
||||
});
|
||||
|
||||
maybeIt('should handle null response', async () => {
|
||||
const result = await extractAndStoreBinaryData(null);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
maybeIt('should handle undefined response', async () => {
|
||||
const result = await extractAndStoreBinaryData(undefined);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Local blob extraction', () => {
|
||||
let mockUploadBlobRemote: ReturnType<typeof vi.fn>;
|
||||
let mockStoreBlob: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetAllMocks();
|
||||
|
||||
// Get the mocked functions
|
||||
const remoteUploadModule = await import('../../src/blobs/remoteUpload');
|
||||
mockUploadBlobRemote = vi.mocked(remoteUploadModule.uploadBlobRemote);
|
||||
|
||||
const blobIndexModule = await import('../../src/blobs/index');
|
||||
mockStoreBlob = vi.mocked(blobIndexModule.storeBlob);
|
||||
|
||||
// Default mock implementations
|
||||
mockStoreBlob.mockResolvedValue({
|
||||
ref: {
|
||||
uri: 'promptfoo://blob/abc123def456',
|
||||
hash: 'abc123def456',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('stores extracted blobs locally without eagerly uploading them', async () => {
|
||||
// Create a large enough data URL to trigger externalization (>1KB)
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const response: ProviderResponse = {
|
||||
output: `data:image/png;base64,${largeBase64}`,
|
||||
};
|
||||
|
||||
await extractAndStoreBinaryData(response);
|
||||
|
||||
// Should store locally
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockUploadBlobRemote).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should externalize image data URIs to blobRefs', async () => {
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const response: ProviderResponse = {
|
||||
output: 'text output',
|
||||
images: [{ data: `data:image/png;base64,${largeBase64}`, mimeType: 'image/png' }],
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
expect(result?.images?.[0].data).toBeUndefined();
|
||||
expect(result?.images?.[0].blobRef).toBeDefined();
|
||||
expect(result?.images?.[0].blobRef?.uri).toContain('promptfoo://blob/');
|
||||
});
|
||||
|
||||
it('should reuse the same image blob when output and images contain identical data URIs', async () => {
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const dataUri = `data:image/png;base64,${largeBase64}`;
|
||||
const response: ProviderResponse = {
|
||||
output: dataUri,
|
||||
images: [{ data: dataUri, mimeType: 'image/png' }],
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.output).toBe(result?.images?.[0].blobRef?.uri);
|
||||
expect(result?.images?.[0].data).toBeUndefined();
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockStoreBlob).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
'image/png',
|
||||
expect.objectContaining({
|
||||
location: 'response.images[0].data',
|
||||
kind: 'image',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should reuse the same image blob for concurrent identical images[] siblings', async () => {
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const dataUri = `data:image/png;base64,${largeBase64}`;
|
||||
const response: ProviderResponse = {
|
||||
output: 'text output',
|
||||
images: [
|
||||
{ data: dataUri, mimeType: 'image/png' },
|
||||
{ data: dataUri, mimeType: 'image/png' },
|
||||
],
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.images?.[0].data).toBeUndefined();
|
||||
expect(result?.images?.[1].data).toBeUndefined();
|
||||
expect(result?.images?.[0].blobRef).toEqual(result?.images?.[1].blobRef);
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should pass through images without data URIs unchanged', async () => {
|
||||
const response: ProviderResponse = {
|
||||
output: 'text output',
|
||||
images: [
|
||||
{
|
||||
blobRef: {
|
||||
uri: 'promptfoo://blob/existing',
|
||||
hash: 'existing',
|
||||
mimeType: 'image/png',
|
||||
sizeBytes: 100,
|
||||
provider: 'local',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
expect(result).toEqual(response);
|
||||
});
|
||||
|
||||
it('should handle mixed images: externalize data URIs but keep existing blobRefs', async () => {
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const response: ProviderResponse = {
|
||||
output: 'text output',
|
||||
images: [
|
||||
{ data: `data:image/png;base64,${largeBase64}`, mimeType: 'image/png' },
|
||||
{
|
||||
blobRef: {
|
||||
uri: 'promptfoo://blob/existing',
|
||||
hash: 'existing',
|
||||
mimeType: 'image/jpeg',
|
||||
sizeBytes: 100,
|
||||
provider: 'local',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
// First image externalized
|
||||
expect(result?.images?.[0].data).toBeUndefined();
|
||||
expect(result?.images?.[0].blobRef).toBeDefined();
|
||||
// Second image kept as-is
|
||||
expect(result?.images?.[1].blobRef?.hash).toBe('existing');
|
||||
});
|
||||
|
||||
it('should record blob references embedded in output text', async () => {
|
||||
const blobIndexModule = await import('../../src/blobs/index');
|
||||
const mockRecordBlobReference = vi.mocked(blobIndexModule.recordBlobReference);
|
||||
const firstHash = 'a'.repeat(64);
|
||||
const secondHash = 'b'.repeat(64);
|
||||
const response: ProviderResponse = {
|
||||
output:
|
||||
`First promptfoo://blob/${firstHash} repeated promptfoo://blob/${firstHash} ` +
|
||||
`second promptfoo://blob/${secondHash}`,
|
||||
};
|
||||
|
||||
await extractAndStoreBinaryData(response, {
|
||||
evalId: 'eval-mixed-output',
|
||||
testIdx: 1,
|
||||
promptIdx: 2,
|
||||
});
|
||||
|
||||
expect(mockRecordBlobReference).toHaveBeenCalledTimes(2);
|
||||
expect(mockRecordBlobReference).toHaveBeenCalledWith(
|
||||
firstHash,
|
||||
expect.objectContaining({
|
||||
evalId: 'eval-mixed-output',
|
||||
testIdx: 1,
|
||||
promptIdx: 2,
|
||||
location: 'response.output',
|
||||
}),
|
||||
);
|
||||
expect(mockRecordBlobReference).toHaveBeenCalledWith(
|
||||
secondHash,
|
||||
expect.objectContaining({
|
||||
evalId: 'eval-mixed-output',
|
||||
testIdx: 1,
|
||||
promptIdx: 2,
|
||||
location: 'response.output',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should reuse the top-level audio blob for mirrored realtime metadata audio data', async () => {
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const response: ProviderResponse = {
|
||||
output: 'test',
|
||||
audio: {
|
||||
data: largeBase64,
|
||||
format: 'wav',
|
||||
transcript: 'hello',
|
||||
},
|
||||
metadata: {
|
||||
audio: {
|
||||
data: largeBase64,
|
||||
format: 'wav',
|
||||
transcript: 'hello',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.audio?.data).toBeUndefined();
|
||||
expect(result?.audio?.blobRef).toBeDefined();
|
||||
expect(result?.metadata?.audio?.data).toBeUndefined();
|
||||
expect(result?.metadata?.audio?.blobRef).toEqual(result?.audio?.blobRef);
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockStoreBlob).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
'audio/wav',
|
||||
expect.objectContaining({
|
||||
location: 'response.audio.data',
|
||||
kind: 'audio',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should externalize metadata-only audio data', async () => {
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const response: ProviderResponse = {
|
||||
output: 'test',
|
||||
metadata: {
|
||||
audio: {
|
||||
data: largeBase64,
|
||||
format: 'wav',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.metadata?.audio?.data).toBeUndefined();
|
||||
expect(result?.metadata?.audio?.blobRef).toBeDefined();
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockStoreBlob).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
'audio/wav',
|
||||
expect.objectContaining({
|
||||
location: 'response.metadata.audio.data',
|
||||
kind: 'audio',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should reuse the same audio blob across turns[].audio and metadata.audio', async () => {
|
||||
// Realtime providers can mirror an audio chunk into both turns[N].audio
|
||||
// and metadata.audio. Without a shared cache, the metadata path stored a
|
||||
// separate blob.
|
||||
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const response = {
|
||||
output: 'test',
|
||||
turns: [
|
||||
{
|
||||
role: 'assistant',
|
||||
audio: { data: largeBase64, format: 'wav', transcript: 'hi' },
|
||||
},
|
||||
],
|
||||
metadata: {
|
||||
audio: { data: largeBase64, format: 'wav', transcript: 'hi' },
|
||||
},
|
||||
} as unknown as ProviderResponse;
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
const turnsBlob = (result as any)?.turns?.[0]?.audio?.blobRef;
|
||||
expect(turnsBlob).toBeDefined();
|
||||
expect(result?.metadata?.audio?.blobRef).toEqual(turnsBlob);
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockStoreBlob).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
'audio/wav',
|
||||
expect.objectContaining({
|
||||
location: 'response.turns[0].audio.data',
|
||||
kind: 'audio',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should reuse the same image blob across images[] and a metadata data-URL of the same bytes', async () => {
|
||||
// E.g. a provider that mirrors a generated image into both `images[]` and
|
||||
// a metadata field. The shared per-response cache canonicalizes on the
|
||||
// parsed bytes so both paths land on one blob.
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const dataUri = `data:image/png;base64,${largeBase64}`;
|
||||
const response: ProviderResponse = {
|
||||
output: 'text',
|
||||
images: [{ data: dataUri, mimeType: 'image/png' }],
|
||||
metadata: {
|
||||
previewImage: dataUri,
|
||||
},
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.images?.[0].blobRef?.uri).toBeDefined();
|
||||
expect(result?.metadata?.previewImage).toBe(result?.images?.[0].blobRef?.uri);
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockStoreBlob).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
'image/png',
|
||||
expect.objectContaining({
|
||||
location: 'response.images[0].data',
|
||||
kind: 'image',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should reuse the same image blob when output is a data URL and metadata holds raw base64', async () => {
|
||||
// Different encodings of identical bytes (data: URL vs. raw base64) must
|
||||
// hit the same cache slot — canonicalization happens on the parsed buffer.
|
||||
const largeBase64 = Buffer.alloc(2000).toString('base64');
|
||||
const dataUri = `data:image/png;base64,${largeBase64}`;
|
||||
const response: ProviderResponse = {
|
||||
output: dataUri,
|
||||
images: [{ data: `data:image/png;base64,${largeBase64}`, mimeType: 'image/png' }],
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.output).toBe(result?.images?.[0].blobRef?.uri);
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should store small raw SVG outputs for the media library without replacing text output', async () => {
|
||||
const svg = '<svg xmlns="http://www.w3.org/2000/svg"><circle r="4"/></svg>';
|
||||
const response: ProviderResponse = { output: svg };
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result?.output).toBe(svg);
|
||||
expect(result?.metadata?.blobUris).toEqual(['promptfoo://blob/abc123def456']);
|
||||
expect(mockStoreBlob).toHaveBeenCalledTimes(1);
|
||||
expect(mockStoreBlob).toHaveBeenCalledWith(
|
||||
Buffer.from(svg),
|
||||
'image/svg+xml',
|
||||
expect.objectContaining({
|
||||
location: 'response.output',
|
||||
kind: 'image',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not store the same raw SVG preview twice when metadata already references it', async () => {
|
||||
const svg = '<svg xmlns="http://www.w3.org/2000/svg"><circle r="4"/></svg>';
|
||||
const response: ProviderResponse = {
|
||||
output: svg,
|
||||
metadata: {
|
||||
blobUris: [`promptfoo://blob/${sha256(Buffer.from(svg))}`],
|
||||
},
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result).toBe(response);
|
||||
expect(mockStoreBlob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not store multiple root SVG documents as one media blob', async () => {
|
||||
const response: ProviderResponse = {
|
||||
output: '<svg><circle/></svg>\n\n<svg><rect/></svg>',
|
||||
};
|
||||
|
||||
const result = await extractAndStoreBinaryData(response);
|
||||
|
||||
expect(result).toBe(response);
|
||||
expect(mockStoreBlob).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { FilesystemBlobStorageProvider } from '../../src/blobs/filesystemProvider';
|
||||
import { createTempDir, removeTempDir } from '../util/utils';
|
||||
|
||||
describe('FilesystemBlobStorageProvider', () => {
|
||||
let tempDir: string | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
removeTempDir(tempDir);
|
||||
tempDir = undefined;
|
||||
});
|
||||
|
||||
it('stores and retrieves blobs by hash', async () => {
|
||||
tempDir = createTempDir('promptfoo-blobs-');
|
||||
const provider = new FilesystemBlobStorageProvider({ basePath: tempDir });
|
||||
|
||||
const data = Buffer.from('blob-data');
|
||||
const { ref } = await provider.store(data, 'application/octet-stream');
|
||||
|
||||
const stored = await provider.getByHash(ref.hash);
|
||||
expect(stored.data.toString('utf8')).toBe('blob-data');
|
||||
});
|
||||
|
||||
it('deduplicates identical blobs', async () => {
|
||||
tempDir = createTempDir('promptfoo-blobs-');
|
||||
const provider = new FilesystemBlobStorageProvider({ basePath: tempDir });
|
||||
|
||||
const data = Buffer.from('same');
|
||||
const first = await provider.store(data, 'application/octet-stream');
|
||||
const second = await provider.store(data, 'application/octet-stream');
|
||||
|
||||
expect(first.ref.hash).toBe(second.ref.hash);
|
||||
expect(second.deduplicated).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects invalid hashes and prevents path traversal', async () => {
|
||||
tempDir = createTempDir('promptfoo-blobs-');
|
||||
const provider = new FilesystemBlobStorageProvider({ basePath: tempDir });
|
||||
|
||||
await expect(provider.exists('../../etc/passwd')).resolves.toBe(false);
|
||||
await expect(provider.getByHash('../../etc/passwd')).rejects.toThrow(/invalid blob hash/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { eq, inArray } from 'drizzle-orm';
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
getShareAuthorizedBlob,
|
||||
isBlobAllowedForShare,
|
||||
recordBlobReference,
|
||||
resetBlobStorageProvider,
|
||||
setBlobStorageProvider,
|
||||
} from '../../src/blobs';
|
||||
import { getDb } from '../../src/database';
|
||||
import { blobAssetsTable, blobReferencesTable, evalsTable } from '../../src/database/tables';
|
||||
import { runDbMigrations } from '../../src/migrate';
|
||||
|
||||
import type { BlobStorageProvider } from '../../src/blobs';
|
||||
|
||||
describe('blob share authorization', () => {
|
||||
const evalId = `eval-${randomUUID()}`;
|
||||
const otherEvalId = `eval-${randomUUID()}`;
|
||||
const trustedHash = 'a'.repeat(64);
|
||||
const unclassifiedHash = 'b'.repeat(64);
|
||||
const importedHash = 'c'.repeat(64);
|
||||
const otherEvalHash = 'd'.repeat(64);
|
||||
const hashes = [trustedHash, unclassifiedHash, importedHash, otherEvalHash];
|
||||
|
||||
beforeAll(async () => {
|
||||
await runDbMigrations();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
const db = await getDb();
|
||||
await db.insert(evalsTable).values([
|
||||
{ id: evalId, config: {}, results: {} },
|
||||
{ id: otherEvalId, config: {}, results: {} },
|
||||
]);
|
||||
await db.insert(blobAssetsTable).values(
|
||||
hashes.map((hash) => ({
|
||||
hash,
|
||||
mimeType: 'image/png',
|
||||
provider: 'filesystem',
|
||||
sizeBytes: 1,
|
||||
})),
|
||||
);
|
||||
await db.insert(blobReferencesTable).values([
|
||||
{
|
||||
id: randomUUID(),
|
||||
blobHash: trustedHash,
|
||||
evalId,
|
||||
kind: 'image',
|
||||
location: 'response.output',
|
||||
},
|
||||
{
|
||||
id: randomUUID(),
|
||||
blobHash: unclassifiedHash,
|
||||
evalId,
|
||||
location: 'response.output',
|
||||
},
|
||||
{
|
||||
id: randomUUID(),
|
||||
blobHash: importedHash,
|
||||
evalId,
|
||||
location: 'import',
|
||||
},
|
||||
{
|
||||
id: randomUUID(),
|
||||
blobHash: otherEvalHash,
|
||||
evalId: otherEvalId,
|
||||
kind: 'image',
|
||||
location: 'response.output',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const db = await getDb();
|
||||
await db
|
||||
.delete(blobReferencesTable)
|
||||
.where(inArray(blobReferencesTable.evalId, [evalId, otherEvalId]));
|
||||
await db.delete(blobAssetsTable).where(inArray(blobAssetsTable.hash, hashes));
|
||||
await db.delete(evalsTable).where(inArray(evalsTable.id, [evalId, otherEvalId]));
|
||||
});
|
||||
|
||||
it('allows only trusted references associated with the local eval', async () => {
|
||||
await expect(isBlobAllowedForShare(trustedHash, evalId)).resolves.toBe(true);
|
||||
await expect(isBlobAllowedForShare(importedHash, evalId)).resolves.toBe(true);
|
||||
await expect(isBlobAllowedForShare(unclassifiedHash, evalId)).resolves.toBe(false);
|
||||
await expect(isBlobAllowedForShare(otherEvalHash, evalId)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('does not authorize a trusted hash after its eval association is removed', async () => {
|
||||
const db = await getDb();
|
||||
await db.delete(blobReferencesTable).where(eq(blobReferencesTable.blobHash, trustedHash));
|
||||
|
||||
await expect(isBlobAllowedForShare(trustedHash, evalId)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('reads local bytes only for share-authorized references', async () => {
|
||||
setBlobStorageProvider({
|
||||
providerId: 'test-stub',
|
||||
store: async () => {
|
||||
throw new Error('not implemented');
|
||||
},
|
||||
getByHash: async (hash: string) => ({
|
||||
data: Buffer.from('trusted-bytes'),
|
||||
metadata: {
|
||||
createdAt: '2026-06-08T00:00:00.000Z',
|
||||
key: hash,
|
||||
mimeType: 'image/png',
|
||||
provider: 'test-stub',
|
||||
sizeBytes: 13,
|
||||
},
|
||||
}),
|
||||
exists: async () => true,
|
||||
deleteByHash: async () => {},
|
||||
getUrl: async () => null,
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(getShareAuthorizedBlob(unclassifiedHash, evalId)).resolves.toBeNull();
|
||||
await expect(getShareAuthorizedBlob(otherEvalHash, evalId)).resolves.toBeNull();
|
||||
|
||||
const blob = await getShareAuthorizedBlob(trustedHash, evalId);
|
||||
expect(blob?.data.toString()).toBe('trusted-bytes');
|
||||
} finally {
|
||||
resetBlobStorageProvider();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('recordBlobReference provenance upgrades', () => {
|
||||
const evalId = `eval-${randomUUID()}`;
|
||||
const hash = 'e'.repeat(64);
|
||||
|
||||
const stubProvider: BlobStorageProvider = {
|
||||
providerId: 'test-stub',
|
||||
store: async () => {
|
||||
throw new Error('not implemented');
|
||||
},
|
||||
getByHash: async () => {
|
||||
throw new Error('not implemented');
|
||||
},
|
||||
exists: async () => true,
|
||||
deleteByHash: async () => {},
|
||||
getUrl: async () => null,
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
await runDbMigrations();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
setBlobStorageProvider(stubProvider);
|
||||
const db = await getDb();
|
||||
await db.insert(evalsTable).values([{ id: evalId, config: {}, results: {} }]);
|
||||
await db
|
||||
.insert(blobAssetsTable)
|
||||
.values([{ hash, mimeType: 'image/png', provider: 'filesystem', sizeBytes: 1 }]);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetBlobStorageProvider();
|
||||
const db = await getDb();
|
||||
await db.delete(blobReferencesTable).where(eq(blobReferencesTable.evalId, evalId));
|
||||
await db.delete(blobAssetsTable).where(eq(blobAssetsTable.hash, hash));
|
||||
await db.delete(evalsTable).where(eq(evalsTable.id, evalId));
|
||||
});
|
||||
|
||||
async function getReferenceRows() {
|
||||
const db = await getDb();
|
||||
return db.select().from(blobReferencesTable).where(eq(blobReferencesTable.evalId, evalId));
|
||||
}
|
||||
|
||||
it('upgrades an unclassified reference once the blob is independently classified', async () => {
|
||||
await recordBlobReference(hash, { evalId, location: 'response.output' });
|
||||
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(false);
|
||||
|
||||
await recordBlobReference(hash, { evalId, kind: 'image', location: 'response.images[0].data' });
|
||||
|
||||
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(true);
|
||||
const rows = await getReferenceRows();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].kind).toBe('image');
|
||||
});
|
||||
|
||||
it('does not authorize re-recorded scan references that carry no kind', async () => {
|
||||
await recordBlobReference(hash, { evalId, location: 'response.output' });
|
||||
await recordBlobReference(hash, { evalId, location: 'response.metadata' });
|
||||
|
||||
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(false);
|
||||
const rows = await getReferenceRows();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].kind).toBeNull();
|
||||
});
|
||||
|
||||
it('does not downgrade a classified reference when re-recorded without a kind', async () => {
|
||||
await recordBlobReference(hash, { evalId, kind: 'image', location: 'response.output' });
|
||||
await recordBlobReference(hash, { evalId, location: 'response.metadata' });
|
||||
|
||||
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(true);
|
||||
const rows = await getReferenceRows();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].kind).toBe('image');
|
||||
});
|
||||
|
||||
it('upgrades a scan reference to import provenance', async () => {
|
||||
await recordBlobReference(hash, { evalId, location: 'response.output' });
|
||||
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(false);
|
||||
|
||||
await recordBlobReference(hash, { evalId, location: 'import' });
|
||||
|
||||
await expect(isBlobAllowedForShare(hash, evalId)).resolves.toBe(true);
|
||||
const rows = await getReferenceRows();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].location).toBe('import');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { shouldAttemptRemoteBlobUpload, uploadBlobRemote } from '../../src/blobs/remoteUpload';
|
||||
import { getEnvBool } from '../../src/envars';
|
||||
import { isLoggedIntoCloud } from '../../src/globalConfig/accounts';
|
||||
import { cloudConfig } from '../../src/globalConfig/cloud';
|
||||
import { fetchWithProxy } from '../../src/util/fetch/index';
|
||||
|
||||
vi.mock('../../src/envars', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('../../src/envars')>();
|
||||
return {
|
||||
...actual,
|
||||
getEnvBool: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../src/globalConfig/accounts', () => ({
|
||||
isLoggedIntoCloud: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../../src/globalConfig/cloud', () => ({
|
||||
cloudConfig: {
|
||||
getApiHost: vi.fn(),
|
||||
getApiKey: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../../src/util/fetch/index', () => ({
|
||||
fetchWithProxy: vi.fn(),
|
||||
}));
|
||||
|
||||
describe('remote blob upload', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(getEnvBool).mockReturnValue(false);
|
||||
vi.mocked(isLoggedIntoCloud).mockReturnValue(true);
|
||||
vi.mocked(cloudConfig.getApiHost).mockReturnValue('https://api.example.com');
|
||||
vi.mocked(cloudConfig.getApiKey).mockReturnValue('test-api-key');
|
||||
vi.mocked(fetchWithProxy).mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
json: async () => ({
|
||||
ref: {
|
||||
uri: 'promptfoo://blob/abc123',
|
||||
hash: 'abc123',
|
||||
},
|
||||
}),
|
||||
} as Response);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('attempts remote upload when sharing is enabled and Cloud auth is configured', () => {
|
||||
expect(shouldAttemptRemoteBlobUpload()).toBe(true);
|
||||
expect(cloudConfig.getApiHost).toHaveBeenCalledTimes(1);
|
||||
expect(cloudConfig.getApiKey).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not attempt remote upload when PROMPTFOO_DISABLE_SHARING is set', async () => {
|
||||
vi.mocked(getEnvBool).mockImplementation((key) => key === 'PROMPTFOO_DISABLE_SHARING');
|
||||
|
||||
expect(shouldAttemptRemoteBlobUpload()).toBe(false);
|
||||
|
||||
const result = await uploadBlobRemote(Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'eval-123',
|
||||
location: 'response.output',
|
||||
kind: 'image',
|
||||
});
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(cloudConfig.getApiHost).not.toHaveBeenCalled();
|
||||
expect(cloudConfig.getApiKey).not.toHaveBeenCalled();
|
||||
expect(fetchWithProxy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('posts blobs to the Cloud blob endpoint when allowed', async () => {
|
||||
await uploadBlobRemote(Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'eval-123',
|
||||
location: 'response.output',
|
||||
kind: 'image',
|
||||
});
|
||||
|
||||
expect(fetchWithProxy).toHaveBeenCalledWith(
|
||||
'https://api.example.com/api/blobs',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
headers: expect.objectContaining({
|
||||
Authorization: 'Bearer test-api-key',
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(cloudConfig.getApiHost).toHaveBeenCalledTimes(1);
|
||||
expect(cloudConfig.getApiKey).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,265 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { getShareAuthorizedBlob } from '../../src/blobs';
|
||||
import { uploadBlobRemote } from '../../src/blobs/remoteUpload';
|
||||
import { createRemoteBlobUploadCache, uploadBlobRefsForShare } from '../../src/blobs/shareUpload';
|
||||
import logger from '../../src/logger';
|
||||
|
||||
import type { StoredBlob } from '../../src/blobs';
|
||||
|
||||
vi.mock('../../src/blobs', () => ({
|
||||
getShareAuthorizedBlob: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../../src/blobs/remoteUpload', () => ({
|
||||
uploadBlobRemote: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../../src/logger', () => ({
|
||||
default: {
|
||||
warn: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const storedBlob: StoredBlob = {
|
||||
data: Buffer.from('image-bytes'),
|
||||
metadata: {
|
||||
createdAt: '2026-06-08T00:00:00.000Z',
|
||||
key: 'blob-key',
|
||||
mimeType: 'image/png',
|
||||
provider: 'filesystem',
|
||||
sizeBytes: 11,
|
||||
},
|
||||
};
|
||||
|
||||
describe('share-time blob upload', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(getShareAuthorizedBlob).mockResolvedValue(storedBlob);
|
||||
vi.mocked(uploadBlobRemote).mockResolvedValue({
|
||||
deduplicated: false,
|
||||
ref: {
|
||||
hash: 'a'.repeat(64),
|
||||
mimeType: 'image/png',
|
||||
provider: 'cloud',
|
||||
sizeBytes: 11,
|
||||
uri: `promptfoo://blob/${'a'.repeat(64)}`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('uploads each referenced blob only once when the share context is unchanged', async () => {
|
||||
const hash = 'a'.repeat(64);
|
||||
const cache = createRemoteBlobUploadCache();
|
||||
|
||||
await uploadBlobRefsForShare(
|
||||
{
|
||||
output: `promptfoo://blob/${hash}`,
|
||||
images: [{ blobRef: { hash, uri: `promptfoo://blob/${hash}` } }],
|
||||
},
|
||||
cache,
|
||||
{ localEvalId: 'local-eval-123', remoteEvalId: 'remote-eval-123', promptIdx: 2, testIdx: 1 },
|
||||
);
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
|
||||
localEvalId: 'local-eval-123',
|
||||
remoteEvalId: 'remote-eval-123',
|
||||
promptIdx: 2,
|
||||
testIdx: 1,
|
||||
});
|
||||
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledWith(hash, 'local-eval-123');
|
||||
expect(uploadBlobRemote).toHaveBeenCalledOnce();
|
||||
expect(uploadBlobRemote).toHaveBeenCalledWith(Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'remote-eval-123',
|
||||
kind: 'image',
|
||||
location: 'share',
|
||||
promptIdx: 2,
|
||||
testIdx: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('uploads the same blob again when it appears in a different result row', async () => {
|
||||
const hash = '7'.repeat(64);
|
||||
const cache = createRemoteBlobUploadCache();
|
||||
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
|
||||
localEvalId: 'local-eval-123',
|
||||
remoteEvalId: 'remote-eval-123',
|
||||
promptIdx: 2,
|
||||
testIdx: 1,
|
||||
});
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
|
||||
localEvalId: 'local-eval-123',
|
||||
remoteEvalId: 'remote-eval-123',
|
||||
promptIdx: 4,
|
||||
testIdx: 3,
|
||||
});
|
||||
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledTimes(2);
|
||||
expect(getShareAuthorizedBlob).toHaveBeenNthCalledWith(1, hash, 'local-eval-123');
|
||||
expect(getShareAuthorizedBlob).toHaveBeenNthCalledWith(2, hash, 'local-eval-123');
|
||||
expect(uploadBlobRemote).toHaveBeenCalledTimes(2);
|
||||
expect(uploadBlobRemote).toHaveBeenNthCalledWith(1, Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'remote-eval-123',
|
||||
kind: 'image',
|
||||
location: 'share',
|
||||
promptIdx: 2,
|
||||
testIdx: 1,
|
||||
});
|
||||
expect(uploadBlobRemote).toHaveBeenNthCalledWith(2, Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'remote-eval-123',
|
||||
kind: 'image',
|
||||
location: 'share',
|
||||
promptIdx: 4,
|
||||
testIdx: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps a zero-index result row distinct from a coordinate-less reference', async () => {
|
||||
// Pins the `?? null` (not `||`) choice in getUploadCacheKey: promptIdx/testIdx
|
||||
// of 0 is the first row of every eval and must not collapse into a reference
|
||||
// that carries no coordinates. A regression to `|| null` would dedupe these two
|
||||
// into a single upload, dropping the (0, 0) row's provenance.
|
||||
const hash = '0'.repeat(64);
|
||||
const cache = createRemoteBlobUploadCache();
|
||||
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
|
||||
localEvalId: 'local-eval-zero',
|
||||
remoteEvalId: 'remote-eval-zero',
|
||||
promptIdx: 0,
|
||||
testIdx: 0,
|
||||
});
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, {
|
||||
localEvalId: 'local-eval-zero',
|
||||
remoteEvalId: 'remote-eval-zero',
|
||||
});
|
||||
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledTimes(2);
|
||||
expect(uploadBlobRemote).toHaveBeenCalledTimes(2);
|
||||
expect(uploadBlobRemote).toHaveBeenNthCalledWith(1, Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'remote-eval-zero',
|
||||
kind: 'image',
|
||||
location: 'share',
|
||||
promptIdx: 0,
|
||||
testIdx: 0,
|
||||
});
|
||||
expect(uploadBlobRemote).toHaveBeenNthCalledWith(2, Buffer.from('image-bytes'), 'image/png', {
|
||||
evalId: 'remote-eval-zero',
|
||||
kind: 'image',
|
||||
location: 'share',
|
||||
promptIdx: undefined,
|
||||
testIdx: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not upload a copied blob URI that is not share-authorized for the eval', async () => {
|
||||
const hash = 'd'.repeat(64);
|
||||
vi.mocked(getShareAuthorizedBlob).mockResolvedValue(null);
|
||||
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
|
||||
localEvalId: 'local-eval-unauthorized',
|
||||
remoteEvalId: 'remote-eval-unauthorized',
|
||||
});
|
||||
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledWith(hash, 'local-eval-unauthorized');
|
||||
expect(uploadBlobRemote).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('logs upload failures without failing the share', async () => {
|
||||
const hash = 'b'.repeat(64);
|
||||
vi.mocked(uploadBlobRemote).mockRejectedValue(new Error('network unavailable'));
|
||||
|
||||
await expect(
|
||||
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
|
||||
localEvalId: 'local-eval-456',
|
||||
remoteEvalId: 'remote-eval-456',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'[Share] Failed to upload referenced blob; shared media may be unavailable',
|
||||
expect.objectContaining({
|
||||
error: 'network unavailable',
|
||||
evalId: 'remote-eval-456',
|
||||
hash,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('logs unavailable remote storage without failing the share', async () => {
|
||||
const hash = 'c'.repeat(64);
|
||||
vi.mocked(uploadBlobRemote).mockResolvedValue(null);
|
||||
|
||||
await expect(
|
||||
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
|
||||
localEvalId: 'local-eval-789',
|
||||
remoteEvalId: 'remote-eval-789',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'[Share] Failed to upload referenced blob; shared media may be unavailable',
|
||||
{
|
||||
evalId: 'remote-eval-789',
|
||||
hash,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('checks authorization only once for repeated unauthorized references', async () => {
|
||||
const hash = 'e'.repeat(64);
|
||||
vi.mocked(getShareAuthorizedBlob).mockResolvedValue(null);
|
||||
const cache = createRemoteBlobUploadCache();
|
||||
const context = { localEvalId: 'local-eval-999', remoteEvalId: 'remote-eval-999' };
|
||||
|
||||
await uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, context);
|
||||
await uploadBlobRefsForShare({ output: `promptfoo://blob/${hash}` }, cache, context);
|
||||
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
|
||||
expect(uploadBlobRemote).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('shares one upload across concurrent references to the same blob', async () => {
|
||||
const hash = 'f'.repeat(64);
|
||||
let releaseAuth: ((blob: StoredBlob | null) => void) | undefined;
|
||||
vi.mocked(getShareAuthorizedBlob).mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
releaseAuth = resolve;
|
||||
}),
|
||||
);
|
||||
const cache = createRemoteBlobUploadCache();
|
||||
const context = { localEvalId: 'local-eval-111', remoteEvalId: 'remote-eval-111' };
|
||||
|
||||
const uploads = Promise.all([
|
||||
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, context),
|
||||
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, cache, context),
|
||||
]);
|
||||
releaseAuth?.(storedBlob);
|
||||
await uploads;
|
||||
|
||||
expect(getShareAuthorizedBlob).toHaveBeenCalledOnce();
|
||||
expect(uploadBlobRemote).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('skips the blob without failing the share when the authorization check errors', async () => {
|
||||
const hash = '9'.repeat(64);
|
||||
vi.mocked(getShareAuthorizedBlob).mockRejectedValue(new Error('db locked'));
|
||||
|
||||
await expect(
|
||||
uploadBlobRefsForShare(`promptfoo://blob/${hash}`, createRemoteBlobUploadCache(), {
|
||||
localEvalId: 'local-eval-222',
|
||||
remoteEvalId: 'remote-eval-222',
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(uploadBlobRemote).not.toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'[Share] Failed to upload referenced blob; shared media may be unavailable',
|
||||
expect.objectContaining({
|
||||
error: 'db locked',
|
||||
hash,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user