promptfoo--promptfoo/test/server/routes/eval.sharing.test.ts

import type { Server } from 'node:http';

import request from 'supertest';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';

// Mock dependencies BEFORE imports
vi.mock('../../../src/node', () => ({
  evaluateWithSource: vi.fn().mockResolvedValue({
    toEvaluateSummary: vi.fn().mockResolvedValue({ results: [] }),
  }),
}));

vi.mock('../../../src/util/sharing', () => ({
  shouldShareResults: vi.fn().mockReturnValue(false),
}));

vi.mock('../../../src/models/eval', () => ({
  default: {
    create: vi.fn(),
    findById: vi.fn(),
  },
}));
vi.mock('../../../src/globalConfig/accounts');

import logger from '../../../src/logger';
import Eval from '../../../src/models/eval';
import { evaluateWithSource } from '../../../src/node';
import { createApp } from '../../../src/server/server';
import { shouldShareResults } from '../../../src/util/sharing';

const errorSpy = vi.spyOn(logger, 'error');
const mockedEvalCreate = vi.mocked(Eval.create);
const mockedEvalFindById = vi.mocked(Eval.findById);
const mockedEvaluateWithSource = vi.mocked(evaluateWithSource);
const mockedShouldShareResults = vi.mocked(shouldShareResults);

describe('Eval Routes - Sharing behavior', () => {
  let api: ReturnType<typeof request.agent>;
  let server: Server;

  beforeAll(async () => {
    await new Promise<void>((resolve, reject) => {
      server = createApp().listen(0, '127.0.0.1', (error?: Error) =>
        error ? reject(error) : resolve(),
      );
    });
    api = request.agent(server);
  });

  afterAll(async () => {
    if (!server.listening) {
      return;
    }
    await new Promise<void>((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  });

  beforeEach(() => {
    vi.resetAllMocks();
    errorSpy.mockClear();

    mockedEvaluateWithSource.mockResolvedValue({
      toEvaluateSummary: vi.fn().mockResolvedValue({ results: [] }),
    } as any);

    mockedShouldShareResults.mockReturnValue(false);
  });

  afterEach(() => {
    vi.resetAllMocks();
  });

  const postJob = (body: Record<string, unknown>) => api.post('/api/eval/job').send(body);

  const minimalTestSuite = {
    prompts: ['test prompt'],
    providers: ['echo'],
    tests: [{ vars: { input: 'test' } }],
  };

  it('should use testSuite.sharing when explicitly set to true', async () => {
    await postJob({ ...minimalTestSuite, sharing: true });

    // Wait for async evaluate call
    await vi.waitFor(() => {
      expect(mockedEvaluateWithSource).toHaveBeenCalled();
    });

    const evaluateArg = mockedEvaluateWithSource.mock.calls[0][0] as any;
    expect(evaluateArg.sharing).toBe(true);
  });

  it('should use testSuite.sharing when explicitly set to false', async () => {
    mockedShouldShareResults.mockReturnValue(true);

    await postJob({ ...minimalTestSuite, sharing: false });

    await vi.waitFor(() => {
      expect(mockedEvaluateWithSource).toHaveBeenCalled();
    });

    const evaluateArg = mockedEvaluateWithSource.mock.calls[0][0] as any;
    expect(evaluateArg.sharing).toBe(false);
  });

  it('should fall back to shouldShareResults when testSuite.sharing is undefined', async () => {
    mockedShouldShareResults.mockReturnValue(true);

    await postJob(minimalTestSuite);

    await vi.waitFor(() => {
      expect(mockedEvaluateWithSource).toHaveBeenCalled();
    });

    const evaluateArg = mockedEvaluateWithSource.mock.calls[0][0] as any;
    expect(evaluateArg.sharing).toBe(true);
    expect(mockedShouldShareResults).toHaveBeenCalledWith({});
  });

  it('does not publish completion before saved results are ready', async () => {
    let resolveSummary: ((summary: { results: never[] }) => void) | undefined;
    mockedEvaluateWithSource.mockResolvedValueOnce({
      id: 'eval-result-id',
      toEvaluateSummary: vi.fn(
        () =>
          new Promise<{ results: never[] }>((resolve) => {
            resolveSummary = resolve;
          }),
      ),
    } as any);

    const createResponse = await postJob(minimalTestSuite);
    const jobId = createResponse.body.id;

    await vi.waitFor(() => {
      expect(resolveSummary).toBeDefined();
    });

    const inProgressResponse = await api.get(`/api/eval/job/${jobId}`);
    expect(inProgressResponse.body.status).toBe('in-progress');
    expect(inProgressResponse.body.evalId).toBeUndefined();

    resolveSummary!({ results: [] });

    await vi.waitFor(async () => {
      const completedResponse = await api.get(`/api/eval/job/${jobId}`);
      expect(completedResponse.body).toMatchObject({
        status: 'complete',
        evalId: 'eval-result-id',
        result: { results: [] },
      });
    });
  });

  it('publishes progress updates while a job is running', async () => {
    let resolveEvaluation: ((value: any) => void) | undefined;
    mockedEvaluateWithSource.mockImplementationOnce(
      (_testSuite, options) =>
        new Promise((resolve) => {
          options?.progressCallback?.(2, 5, 0, {} as never, {} as never);
          resolveEvaluation = resolve;
        }),
    );

    const createResponse = await postJob(minimalTestSuite);
    const jobId = createResponse.body.id;

    const inProgressResponse = await api.get(`/api/eval/job/${jobId}`);
    expect(inProgressResponse.body).toMatchObject({
      status: 'in-progress',
      progress: 2,
      total: 5,
    });

    resolveEvaluation!({
      id: 'eval-result-id',
      toEvaluateSummary: vi.fn().mockResolvedValue({ results: [] }),
    });

    await vi.waitFor(async () => {
      const completedResponse = await api.get(`/api/eval/job/${jobId}`);
      expect(completedResponse.body.status).toBe('complete');
    });
  });

  it('flips status to error when toEvaluateSummary rejects', async () => {
    mockedEvaluateWithSource.mockResolvedValueOnce({
      id: 'eval-result-id',
      toEvaluateSummary: vi.fn().mockRejectedValue(new Error('summary boom')),
    } as any);

    const createResponse = await postJob(minimalTestSuite);
    const jobId = createResponse.body.id;

    await vi.waitFor(async () => {
      const response = await api.get(`/api/eval/job/${jobId}`);
      expect(response.body.status).toBe('error');
    });
  });

  it('should not log the raw job body on evaluation failure', async () => {
    mockedEvaluateWithSource.mockRejectedValueOnce(new Error('boom'));

    const sensitiveBody = {
      prompts: ['test prompt'],
      providers: ['echo'],
      tests: [{ vars: { input: 'secret value', apiKey: 'sk-test-12345678901234567890' } }],
    };

    await postJob(sensitiveBody);

    await vi.waitFor(() => {
      expect(errorSpy).toHaveBeenCalled();
    });

    expect(errorSpy).toHaveBeenCalledWith(
      'Failed to eval tests',
      expect.objectContaining({
        body: expect.objectContaining({
          prompts: ['test prompt'],
          providers: ['echo'],
          tests: [
            expect.objectContaining({
              vars: expect.objectContaining({ apiKey: '[REDACTED]' }),
            }),
          ],
        }),
      }),
    );
    expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('sk-test-12345678901234567890');
  });

  it('restores redacted Azure SAS tokens before rerunning a stored eval', async () => {
    const sasUri = 'az://account/container/tests.yaml?sp=r&sig=azure-secret';
    mockedEvalFindById.mockResolvedValueOnce({ config: { tests: sasUri } } as never);

    await postJob({
      ...minimalTestSuite,
      tests: 'az://account/container/tests.yaml?sp=r&sig=%5BREDACTED%5D',
      sourceEvalId: 'source-eval-id',
    });

    await vi.waitFor(() => {
      expect(mockedEvaluateWithSource).toHaveBeenCalled();
    });

    const evaluateArg = mockedEvaluateWithSource.mock.calls[0][0] as any;
    expect(evaluateArg.tests).toBe(sasUri);
  });

  it('should not log the raw save body on database failure', async () => {
    mockedEvalCreate.mockRejectedValueOnce(new Error('db down'));

    const secret = 'sk-test-12345678901234567890';
    const response = await api.post('/api/eval').send({
      config: {
        providers: [{ id: 'openai:gpt-4o', config: { apiKey: secret } }],
      },
      prompts: [{ raw: 'test prompt', label: 'test prompt' }],
      results: [{ promptIdx: 0, testIdx: 0, success: true, score: 1 }],
    });

    expect(response.status).toBe(500);
    expect(errorSpy).toHaveBeenCalledWith(
      'Failed to write eval to database',
      expect.objectContaining({
        body: expect.objectContaining({
          config: expect.objectContaining({
            providers: [
              expect.objectContaining({
                config: expect.objectContaining({ apiKey: '[REDACTED]' }),
              }),
            ],
          }),
        }),
      }),
    );
    expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(secret);
  });

  it('should return false from shouldShareResults when cloud is disabled', async () => {
    mockedShouldShareResults.mockReturnValue(false);

    await postJob(minimalTestSuite);

    await vi.waitFor(() => {
      expect(mockedEvaluateWithSource).toHaveBeenCalled();
    });

    const evaluateArg = mockedEvaluateWithSource.mock.calls[0][0] as any;
    expect(evaluateArg.sharing).toBe(false);
  });
});