chore: import upstream snapshot with attribution

apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts

@@ -0,0 +1,499 @@
+import '@sim/testing/mocks/executor'
+
+import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+
+vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock)
+
+vi.mock('@/lib/credentials/access', () => ({
+  getCredentialActorContext: vi.fn().mockResolvedValue({
+    credential: {
+      id: 'test-vertex-credential-id',
+      type: 'oauth',
+      workspaceId: 'test-workspace',
+      accountId: 'test-vertex-credential-id',
+    },
+    member: { role: 'admin', status: 'active' },
+    hasWorkspaceAccess: true,
+    canWriteWorkspace: true,
+    isAdmin: true,
+  }),
+}))
+
+import { BlockType } from '@/executor/constants'
+import { EvaluatorBlockHandler } from '@/executor/handlers/evaluator/evaluator-handler'
+import type { ExecutionContext } from '@/executor/types'
+import { getProviderFromModel } from '@/providers/utils'
+import type { SerializedBlock } from '@/serializer/types'
+
+const mockGetProviderFromModel = getProviderFromModel as Mock
+const mockFetch = global.fetch as unknown as Mock
+
+describe('EvaluatorBlockHandler', () => {
+  let handler: EvaluatorBlockHandler
+  let mockBlock: SerializedBlock
+  let mockContext: ExecutionContext
+
+  beforeEach(() => {
+    handler = new EvaluatorBlockHandler()
+
+    mockBlock = {
+      id: 'eval-block-1',
+      metadata: { id: BlockType.EVALUATOR, name: 'Test Evaluator' },
+      position: { x: 20, y: 20 },
+      config: { tool: BlockType.EVALUATOR, params: {} },
+      inputs: {
+        content: 'string',
+        metrics: 'json',
+        model: 'string',
+        temperature: 'number',
+      }, // Using ParamType strings
+      outputs: {},
+      enabled: true,
+    }
+
+    mockContext = {
+      workflowId: 'test-workflow-id',
+      userId: 'test-user',
+      blockStates: new Map(),
+      blockLogs: [],
+      metadata: { duration: 0 },
+      environmentVariables: {},
+      decisions: { router: new Map(), condition: new Map() },
+      loopExecutions: new Map(),
+      completedLoops: new Set(),
+      executedBlocks: new Set(),
+      activeExecutionPath: new Set(),
+    }
+
+    // Reset mocks using vi
+    vi.clearAllMocks()
+
+    // Default mock implementations
+    authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValue({
+      accountId: 'test-vertex-credential-id',
+      usedCredentialTable: false,
+    })
+    authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValue({
+      accessToken: 'mock-access-token',
+      refreshed: false,
+    })
+    mockGetProviderFromModel.mockReturnValue('openai')
+
+    // Set up fetch mock to return a successful response
+    mockFetch.mockImplementation(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ score1: 5, score2: 8 }),
+            model: 'mock-model',
+            tokens: { input: 50, output: 10, total: 60 },
+            cost: 0.002,
+            timing: { total: 200 },
+          }),
+      })
+    })
+  })
+
+  it('should handle evaluator blocks', () => {
+    expect(handler.canHandle(mockBlock)).toBe(true)
+    const nonEvalBlock: SerializedBlock = { ...mockBlock, metadata: { id: 'other' } }
+    expect(handler.canHandle(nonEvalBlock)).toBe(false)
+  })
+
+  it('should execute evaluator block correctly with basic inputs', async () => {
+    const inputs = {
+      content: 'This is the content to evaluate.',
+      metrics: [
+        { name: 'score1', description: 'First score', range: { min: 0, max: 10 } },
+        { name: 'score2', description: 'Second score', range: { min: 0, max: 10 } },
+      ],
+      model: 'gpt-4o',
+      apiKey: 'test-api-key',
+      temperature: 0.1,
+    }
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect(mockGetProviderFromModel).toHaveBeenCalledWith('gpt-4o')
+    expect(mockFetch).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        method: 'POST',
+        headers: expect.any(Object),
+        body: expect.any(String),
+      })
+    )
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+    expect(requestBody).toMatchObject({
+      provider: 'openai',
+      model: 'gpt-4o',
+      systemPrompt: expect.stringContaining(inputs.content),
+      responseFormat: expect.objectContaining({
+        schema: {
+          type: 'object',
+          properties: {
+            score1: { type: 'number' },
+            score2: { type: 'number' },
+          },
+          required: ['score1', 'score2'],
+          additionalProperties: false,
+        },
+      }),
+      temperature: 0.1,
+    })
+
+    expect(result).toEqual({
+      content: 'This is the content to evaluate.',
+      model: 'mock-model',
+      tokens: { input: 50, output: 10, total: 60 },
+      cost: {
+        input: 0,
+        output: 0,
+        total: 0,
+      },
+      score1: 5,
+      score2: 8,
+    })
+  })
+
+  it('should process JSON string content correctly', async () => {
+    const contentObj = { text: 'Evaluate this JSON.', value: 42 }
+    const inputs = {
+      content: JSON.stringify(contentObj),
+      metrics: [{ name: 'clarity', description: 'Clarity score', range: { min: 1, max: 5 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ clarity: 4 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+    expect(requestBody).toMatchObject({
+      systemPrompt: expect.stringContaining(JSON.stringify(contentObj, null, 2)),
+    })
+  })
+
+  it('should process object content correctly', async () => {
+    const contentObj = { data: [1, 2, 3], status: 'ok' }
+    const inputs = {
+      content: contentObj,
+      metrics: [
+        { name: 'completeness', description: 'Data completeness', range: { min: 0, max: 1 } },
+      ],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ completeness: 1 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+    expect(requestBody).toMatchObject({
+      systemPrompt: expect.stringContaining(JSON.stringify(contentObj, null, 2)),
+    })
+  })
+
+  it('should parse valid JSON response correctly', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: '```json\n{ "quality": 9 }\n```',
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).quality).toBe(9)
+  })
+
+  it('should handle invalid/non-JSON response gracefully (scores = 0)', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 5 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: 'Sorry, I cannot provide a score.',
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).score).toBe(0)
+  })
+
+  it('should handle partially valid JSON response (extracts what it can)', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [
+        { name: 'accuracy', description: 'Acc', range: { min: 0, max: 1 } },
+        { name: 'fluency', description: 'Flu', range: { min: 0, max: 1 } },
+      ],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: '{ "accuracy": 1, "fluency": invalid }',
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+    expect((result as any).accuracy).toBe(0)
+    expect((result as any).fluency).toBe(0)
+  })
+
+  it('should extract metric scores ignoring case', async () => {
+    const inputs = {
+      content: 'Test',
+      metrics: [{ name: 'CamelCaseScore', description: 'Desc', range: { min: 0, max: 10 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ camelcasescore: 7 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).camelcasescore).toBe(7)
+  })
+
+  it('should handle missing metrics in response (score = 0)', async () => {
+    const inputs = {
+      content: 'Test',
+      metrics: [
+        { name: 'presentScore', description: 'Desc1', range: { min: 0, max: 5 } },
+        { name: 'missingScore', description: 'Desc2', range: { min: 0, max: 5 } },
+      ],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ presentScore: 4 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).presentscore).toBe(4)
+    expect((result as any).missingscore).toBe(0)
+  })
+
+  it('should handle server error responses', async () => {
+    const inputs = { content: 'Test error handling.', apiKey: 'test-api-key' }
+
+    // Override fetch mock to return an error
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: false,
+        status: 500,
+        json: () => Promise.resolve({ error: 'Server error' }),
+      })
+    })
+
+    await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow('Server error')
+  })
+
+  it('should handle Azure OpenAI models with endpoint and API version', async () => {
+    const inputs = {
+      content: 'Test content to evaluate',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      model: 'gpt-4o',
+      apiKey: 'test-azure-key',
+      azureEndpoint: 'https://test.openai.azure.com',
+      azureApiVersion: '2024-07-01-preview',
+    }
+
+    mockGetProviderFromModel.mockReturnValue('azure-openai')
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ quality: 8 }),
+            model: 'gpt-4o',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody).toMatchObject({
+      provider: 'azure-openai',
+      model: 'gpt-4o',
+      apiKey: 'test-azure-key',
+      azureEndpoint: 'https://test.openai.azure.com',
+      azureApiVersion: '2024-07-01-preview',
+    })
+  })
+
+  it('should handle Vertex AI models with OAuth credential', async () => {
+    const inputs = {
+      content: 'Test content to evaluate',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      model: 'gemini-2.0-flash-exp',
+      vertexCredential: 'test-vertex-credential-id',
+      vertexProject: 'test-gcp-project',
+      vertexLocation: 'us-central1',
+    }
+
+    mockGetProviderFromModel.mockReturnValue('vertex')
+
+    // Mock the database query for Vertex credential
+    const mockDb = await import('@sim/db')
+    const mockAccount = {
+      id: 'test-vertex-credential-id',
+      accessToken: 'mock-access-token',
+      refreshToken: 'mock-refresh-token',
+      expiresAt: new Date(Date.now() + 3600000), // 1 hour from now
+    }
+    ;(mockDb.db.query as any).account = { findFirst: vi.fn() }
+    vi.spyOn(mockDb.db.query.account, 'findFirst').mockResolvedValue(mockAccount as any)
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ quality: 9 }),
+            model: 'gemini-2.0-flash-exp',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody).toMatchObject({
+      provider: 'vertex',
+      model: 'gemini-2.0-flash-exp',
+      vertexProject: 'test-gcp-project',
+      vertexLocation: 'us-central1',
+    })
+    expect(requestBody.apiKey).toBe('mock-access-token')
+  })
+
+  it('should use default model when not provided', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 10 } }],
+      apiKey: 'test-api-key',
+      // No model provided - should use default
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ score: 7 }),
+            model: 'claude-sonnet-5',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody.model).toBe('claude-sonnet-5')
+  })
+})