promptfoo--promptfoo/test/integration/function-provider-grading.test.ts

/**
 * Integration test to verify function providers work end-to-end in grading scenarios
 * This tests the exact issue from #3784: function providers in defaultTest.options.provider
 */
import { describe, expect, it, vi } from 'vitest';
import { getGradingProvider } from '../../src/matchers/providers';
import { createMockProvider } from '../factories/provider';

import type { ApiProvider, ProviderType } from '../../src/types/providers';

describe('Function Provider Integration - Issue #3784', () => {
  it('should work with getGradingProvider when passed as ApiProvider object', async () => {
    // Create a mock function provider (simulating what resolveProvider returns)
    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
      return { output: `Graded: ${prompt}` };
    });
    mockFunctionProvider.label = 'test-grader';

    // This is what resolveProvider returns for function providers.
    // Use a plain object here because the test asserts callApi identity.
    const resolvedProvider: ApiProvider = {
      id: () => mockFunctionProvider.label,
      callApi: mockFunctionProvider,
    };

    // Now pass it through getGradingProvider (this is what matcher provider helpers do)
    const gradingProvider = await getGradingProvider(
      'text' as ProviderType,
      resolvedProvider,
      null,
    );

    expect(gradingProvider).toBeDefined();
    expect(gradingProvider).toBe(resolvedProvider); // Should return same object
    expect(typeof gradingProvider!.id).toBe('function');
    expect(gradingProvider!.id()).toBe('test-grader');
    expect(gradingProvider!.callApi).toBe(mockFunctionProvider);
  });

  it('should actually call the function provider', async () => {
    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
      return { output: `Response for: ${prompt}` };
    });

    const resolvedProvider = createMockProvider({
      id: 'custom-grader',
      callApi: mockFunctionProvider,
    });

    const gradingProvider = await getGradingProvider(
      'text' as ProviderType,
      resolvedProvider,
      null,
    );

    // Actually call the provider
    const result = await gradingProvider!.callApi('test prompt');

    expect(mockFunctionProvider).toHaveBeenCalledWith('test prompt');
    expect(result.output).toBe('Response for: test prompt');
  });

  it('should handle function provider without label', async () => {
    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
      return { output: `Graded: ${prompt}` };
    });

    // No label, so resolveProvider uses 'custom-function'
    const resolvedProvider = createMockProvider({
      id: 'custom-function',
      callApi: mockFunctionProvider,
    });

    const gradingProvider = await getGradingProvider(
      'text' as ProviderType,
      resolvedProvider,
      null,
    );

    expect(gradingProvider).toBeDefined();
    expect(gradingProvider!.id()).toBe('custom-function');
  });

  it('should correctly identify as ApiProvider based on type check', async () => {
    const mockFunctionProvider: any = vi.fn(async () => ({ output: 'test' }));

    const resolvedProvider = createMockProvider({
      id: 'test',
      callApi: mockFunctionProvider,
    });

    // This is the exact check from getGradingProvider line 120
    const isApiProviderCheck =
      typeof resolvedProvider === 'object' &&
      typeof (resolvedProvider as ApiProvider).id === 'function';

    expect(isApiProviderCheck).toBe(true);
  });
});