chore: import upstream snapshot with attribution

test/util/jsonExport.test.ts

@@ -0,0 +1,408 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import * as blobs from '../../src/blobs';
+import { getTraceStore } from '../../src/tracing/store';
+import { writeOutput } from '../../src/util/index';
+import { createTempDir, mockProcessEnv, removeTempDir } from './utils';
+
+// Mock dependencies
+vi.mock('../../src/database', () => ({
+  getDb: vi.fn().mockReturnValue({
+    select: vi.fn(),
+    insert: vi.fn(),
+    transaction: vi.fn(),
+  }),
+}));
+
+vi.mock('../../src/logger', () => ({
+  default: {
+    info: vi.fn(),
+    debug: vi.fn(),
+    error: vi.fn(),
+    warn: vi.fn(),
+  },
+}));
+
+describe('JSON export with improved error handling', () => {
+  let tempDir: string;
+  let tempFilePath: string;
+  let mockEval: any;
+
+  beforeEach(() => {
+    tempDir = createTempDir('promptfoo-json-test-');
+    tempFilePath = path.join(tempDir, 'test-export.json');
+
+    mockEval = {
+      id: 'test-eval-id',
+      createdAt: '2025-01-01T00:00:00.000Z',
+      author: 'test-author',
+      config: { testConfig: true },
+      prompts: [
+        { raw: 'Test prompt 1', label: 'prompt1' },
+        { raw: 'Test prompt 2', label: 'prompt2' },
+      ],
+      toEvaluateSummary: vi.fn(),
+      getResultsCount: vi.fn(),
+    };
+  });
+
+  afterEach(() => {
+    removeTempDir(tempDir);
+    vi.clearAllMocks();
+  });
+
+  describe('normal JSON export', () => {
+    beforeEach(() => {
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: [
+          { testIdx: 0, promptIdx: 0, success: true, score: 1.0 },
+          { testIdx: 1, promptIdx: 0, success: true, score: 0.9 },
+        ],
+        stats: {
+          successes: 2,
+          failures: 0,
+          tokenUsage: { total: 100, prompt: 50, completion: 50 },
+        },
+      });
+    });
+
+    it('should export JSON successfully', async () => {
+      await writeOutput(tempFilePath, mockEval, 'https://share.url');
+
+      expect(fs.existsSync(tempFilePath)).toBe(true);
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      // Verify structure matches expected OutputFile format
+      expect(parsed).toHaveProperty('evalId', 'test-eval-id');
+      expect(parsed).toHaveProperty('results');
+      expect(parsed.results).toHaveProperty('version', 3);
+      expect(parsed.results.results).toHaveLength(2);
+      expect(parsed).toHaveProperty('config', { testConfig: true });
+      expect(parsed).toHaveProperty('shareableUrl', 'https://share.url');
+      expect(parsed).toHaveProperty('metadata');
+    });
+
+    it('should export persisted vars and runtime options for round-trip imports', async () => {
+      mockEval.vars = ['topic', 'tone'];
+      mockEval.runtimeOptions = { cache: false, maxConcurrency: 2 };
+
+      await writeOutput(tempFilePath, mockEval, null);
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed).toHaveProperty('vars', ['topic', 'tone']);
+      expect(parsed).toHaveProperty('runtimeOptions', { cache: false, maxConcurrency: 2 });
+    });
+
+    it('should embed referenced blob bytes only when media export is enabled', async () => {
+      const hash = 'a'.repeat(64);
+      const data = Buffer.from('portable image');
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: [{ response: { output: `promptfoo://blob/${hash}` } }],
+        stats: { successes: 1, failures: 0 },
+      });
+      vi.spyOn(blobs, 'getBlobByHash').mockResolvedValue({
+        data,
+        metadata: {
+          mimeType: 'image/png',
+          sizeBytes: data.length,
+          createdAt: '2025-01-01T00:00:00.000Z',
+          provider: 'filesystem',
+          key: hash,
+        },
+      });
+
+      await writeOutput(tempFilePath, mockEval, null);
+      expect(JSON.parse(fs.readFileSync(tempFilePath, 'utf8'))).not.toHaveProperty('blobAssets');
+
+      await writeOutput(tempFilePath, mockEval, null, { includeMedia: true });
+      const parsed = JSON.parse(fs.readFileSync(tempFilePath, 'utf8'));
+      expect(parsed.blobAssets).toEqual([
+        {
+          hash,
+          mimeType: 'image/png',
+          sizeBytes: data.length,
+          data: data.toString('base64'),
+        },
+      ]);
+    });
+
+    it('should embed blob bytes referenced only by exported traces', async () => {
+      const hash = 'b'.repeat(64);
+      const data = Buffer.from('portable trace image');
+      const traceSpy = vi.spyOn(getTraceStore(), 'getTracesByEvaluation').mockResolvedValue([
+        {
+          traceId: 'trace-media-export',
+          evaluationId: mockEval.id,
+          testCaseId: 'trace-media-case',
+          metadata: { attachment: `promptfoo://blob/${hash}` },
+          spans: [],
+        },
+      ]);
+      vi.spyOn(blobs, 'getBlobByHash').mockResolvedValue({
+        data,
+        metadata: {
+          mimeType: 'image/png',
+          sizeBytes: data.length,
+          createdAt: '2025-01-01T00:00:00.000Z',
+          provider: 'filesystem',
+          key: hash,
+        },
+      });
+
+      try {
+        await writeOutput(tempFilePath, mockEval, null, { includeMedia: true });
+
+        const parsed = JSON.parse(fs.readFileSync(tempFilePath, 'utf8'));
+        expect(parsed.traces[0].metadata.attachment).toBe(`promptfoo://blob/${hash}`);
+        expect(parsed.blobAssets).toEqual([
+          {
+            hash,
+            mimeType: 'image/png',
+            sizeBytes: data.length,
+            data: data.toString('base64'),
+          },
+        ]);
+      } finally {
+        traceSpy.mockRestore();
+      }
+    });
+
+    it('should not embed response blob bytes when response output stripping is enabled', async () => {
+      const restoreEnv = mockProcessEnv({ PROMPTFOO_STRIP_RESPONSE_OUTPUT: 'true' });
+      const hash = 'c'.repeat(64);
+      const getBlobSpy = vi.spyOn(blobs, 'getBlobByHash');
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: [
+          {
+            response: {
+              output: '[output stripped]',
+              metadata: { blobUris: [`promptfoo://blob/${hash}`] },
+            },
+          },
+        ],
+        stats: { successes: 1, failures: 0 },
+      });
+
+      try {
+        await writeOutput(tempFilePath, mockEval, null, { includeMedia: true });
+
+        const parsed = JSON.parse(fs.readFileSync(tempFilePath, 'utf8'));
+        expect(parsed.results.results[0].response.metadata.blobUris).toEqual([
+          `promptfoo://blob/${hash}`,
+        ]);
+        expect(parsed).not.toHaveProperty('blobAssets');
+        expect(getBlobSpy).not.toHaveBeenCalled();
+      } finally {
+        getBlobSpy.mockRestore();
+        restoreEnv();
+      }
+    });
+
+    it('should handle null shareableUrl', async () => {
+      await writeOutput(tempFilePath, mockEval, null);
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.shareableUrl).toBeNull();
+    });
+
+    it('should maintain proper JSON formatting', async () => {
+      await writeOutput(tempFilePath, mockEval, 'https://test.url');
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+
+      // Verify proper 2-space indentation
+      expect(content).toContain('{\n  "evalId":');
+      expect(content).toContain('  "results": {');
+      expect(content).toContain('    "version":');
+
+      // Should be valid JSON
+      expect(() => JSON.parse(content)).not.toThrow();
+    });
+
+    it('should include all required metadata fields', async () => {
+      await writeOutput(tempFilePath, mockEval, 'https://test.url');
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.metadata).toHaveProperty('promptfooVersion');
+      expect(parsed.metadata).toHaveProperty('nodeVersion');
+      expect(parsed.metadata).toHaveProperty('platform');
+      expect(parsed.metadata).toHaveProperty('exportedAt');
+      expect(parsed.metadata).toHaveProperty('author', 'test-author');
+    });
+  });
+
+  describe('memory limit error handling', () => {
+    beforeEach(() => {
+      mockEval.getResultsCount.mockResolvedValue(50000);
+      mockEval.toEvaluateSummary.mockImplementation(() => {
+        throw new RangeError('Invalid string length');
+      });
+    });
+
+    it('should handle RangeError gracefully with helpful message', async () => {
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+        'Dataset too large for JSON export',
+      );
+
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+        'Consider using JSONL format instead',
+      );
+    });
+
+    it('should include result count in error message', async () => {
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow('50000 results');
+    });
+
+    it('should not create output file when memory error occurs', async () => {
+      try {
+        await writeOutput(tempFilePath, mockEval, null);
+      } catch (_error) {
+        // Expected to throw
+      }
+
+      expect(fs.existsSync(tempFilePath)).toBe(false);
+    });
+  });
+
+  describe('other error handling', () => {
+    it('should propagate non-RangeError exceptions', async () => {
+      const testError = new Error('Database connection failed');
+      mockEval.toEvaluateSummary.mockRejectedValue(testError);
+
+      await expect(writeOutput(tempFilePath, mockEval, null)).rejects.toThrow(
+        'Database connection failed',
+      );
+    });
+
+    it('should handle file system errors', async () => {
+      // Use a path that will definitely fail - writing to a file that already exists as a directory
+      const dirAsFile = path.join(tempDir, 'directory-as-file');
+      fs.mkdirSync(dirAsFile);
+      const invalidPath = dirAsFile; // Try to write to a directory path as if it were a file
+
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        results: [],
+        stats: { successes: 0, failures: 0 },
+      });
+
+      await expect(writeOutput(invalidPath, mockEval, null)).rejects.toThrow();
+    });
+  });
+
+  describe('backward compatibility', () => {
+    it('should maintain exact same JSON structure as original implementation', async () => {
+      const expectedSummary = {
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: [
+          { raw: 'Test prompt 1', label: 'prompt1' },
+          { raw: 'Test prompt 2', label: 'prompt2' },
+        ],
+        results: [
+          {
+            testIdx: 0,
+            promptIdx: 0,
+            success: true,
+            score: 1.0,
+            vars: { input: 'test' },
+            output: 'response',
+          },
+        ],
+        stats: {
+          successes: 1,
+          failures: 0,
+          errors: 0,
+          tokenUsage: { total: 100, prompt: 50, completion: 50 },
+        },
+      };
+
+      mockEval.toEvaluateSummary.mockResolvedValue(expectedSummary);
+
+      await writeOutput(tempFilePath, mockEval, 'https://share.url');
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      // Verify exact structure
+      expect(Object.keys(parsed).sort()).toEqual(
+        ['evalId', 'results', 'config', 'shareableUrl', 'metadata'].sort(),
+      );
+
+      expect(parsed.results).toEqual(expectedSummary);
+    });
+
+    it('should handle both EvaluateSummaryV2 and V3 formats', async () => {
+      const v2Summary = {
+        version: 2,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        results: [{ testIdx: 0, success: true }],
+        table: { head: { prompts: [], vars: [] }, body: [] },
+        stats: { successes: 1, failures: 0 },
+      };
+
+      mockEval.toEvaluateSummary.mockResolvedValue(v2Summary);
+
+      await writeOutput(tempFilePath, mockEval, null);
+
+      const content = fs.readFileSync(tempFilePath, 'utf8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.results.version).toBe(2);
+      expect(parsed.results).toHaveProperty('table');
+      expect(parsed.results.results).toEqual([{ testIdx: 0, success: true }]);
+    });
+  });
+
+  describe('performance considerations', () => {
+    it('should complete export in reasonable time for moderate datasets', async () => {
+      // Create a moderate-sized dataset
+      const moderateResults = Array.from({ length: 1000 }, (_, i) => ({
+        testIdx: i,
+        promptIdx: 0,
+        success: true,
+        score: Math.random(),
+        vars: { input: `test input ${i}` },
+        output: `test output ${i}`,
+      }));
+
+      mockEval.toEvaluateSummary.mockResolvedValue({
+        version: 3,
+        timestamp: '2025-01-01T00:00:00.000Z',
+        prompts: mockEval.prompts,
+        results: moderateResults,
+        stats: { successes: 1000, failures: 0 },
+      });
+
+      const startTime = Date.now();
+      await writeOutput(tempFilePath, mockEval, null);
+      const duration = Date.now() - startTime;
+
+      // Should complete within reasonable time (5 seconds for 1000 results)
+      expect(duration).toBeLessThan(5000);
+
+      // Verify file was created and has content
+      expect(fs.existsSync(tempFilePath)).toBe(true);
+      const stats = fs.statSync(tempFilePath);
+      expect(stats.size).toBeGreaterThan(1000); // Should have substantial content
+    }, 10000); // 10 second timeout
+  });
+});