chore: import upstream snapshot with attribution

test/integration/function-provider-grading.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Integration test to verify function providers work end-to-end in grading scenarios
+ * This tests the exact issue from #3784: function providers in defaultTest.options.provider
+ */
+import { describe, expect, it, vi } from 'vitest';
+import { getGradingProvider } from '../../src/matchers/providers';
+import { createMockProvider } from '../factories/provider';
+
+import type { ApiProvider, ProviderType } from '../../src/types/providers';
+
+describe('Function Provider Integration - Issue #3784', () => {
+  it('should work with getGradingProvider when passed as ApiProvider object', async () => {
+    // Create a mock function provider (simulating what resolveProvider returns)
+    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
+      return { output: `Graded: ${prompt}` };
+    });
+    mockFunctionProvider.label = 'test-grader';
+
+    // This is what resolveProvider returns for function providers.
+    // Use a plain object here because the test asserts callApi identity.
+    const resolvedProvider: ApiProvider = {
+      id: () => mockFunctionProvider.label,
+      callApi: mockFunctionProvider,
+    };
+
+    // Now pass it through getGradingProvider (this is what matcher provider helpers do)
+    const gradingProvider = await getGradingProvider(
+      'text' as ProviderType,
+      resolvedProvider,
+      null,
+    );
+
+    expect(gradingProvider).toBeDefined();
+    expect(gradingProvider).toBe(resolvedProvider); // Should return same object
+    expect(typeof gradingProvider!.id).toBe('function');
+    expect(gradingProvider!.id()).toBe('test-grader');
+    expect(gradingProvider!.callApi).toBe(mockFunctionProvider);
+  });
+
+  it('should actually call the function provider', async () => {
+    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
+      return { output: `Response for: ${prompt}` };
+    });
+
+    const resolvedProvider = createMockProvider({
+      id: 'custom-grader',
+      callApi: mockFunctionProvider,
+    });
+
+    const gradingProvider = await getGradingProvider(
+      'text' as ProviderType,
+      resolvedProvider,
+      null,
+    );
+
+    // Actually call the provider
+    const result = await gradingProvider!.callApi('test prompt');
+
+    expect(mockFunctionProvider).toHaveBeenCalledWith('test prompt');
+    expect(result.output).toBe('Response for: test prompt');
+  });
+
+  it('should handle function provider without label', async () => {
+    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
+      return { output: `Graded: ${prompt}` };
+    });
+
+    // No label, so resolveProvider uses 'custom-function'
+    const resolvedProvider = createMockProvider({
+      id: 'custom-function',
+      callApi: mockFunctionProvider,
+    });
+
+    const gradingProvider = await getGradingProvider(
+      'text' as ProviderType,
+      resolvedProvider,
+      null,
+    );
+
+    expect(gradingProvider).toBeDefined();
+    expect(gradingProvider!.id()).toBe('custom-function');
+  });
+
+  it('should correctly identify as ApiProvider based on type check', async () => {
+    const mockFunctionProvider: any = vi.fn(async () => ({ output: 'test' }));
+
+    const resolvedProvider = createMockProvider({
+      id: 'test',
+      callApi: mockFunctionProvider,
+    });
+
+    // This is the exact check from getGradingProvider line 120
+    const isApiProviderCheck =
+      typeof resolvedProvider === 'object' &&
+      typeof (resolvedProvider as ApiProvider).id === 'function';
+
+    expect(isApiProviderCheck).toBe(true);
+  });
+});