chore: import upstream snapshot with attribution

apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts

@@ -0,0 +1,499 @@
+import '@sim/testing/mocks/executor'
+
+import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+
+vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock)
+
+vi.mock('@/lib/credentials/access', () => ({
+  getCredentialActorContext: vi.fn().mockResolvedValue({
+    credential: {
+      id: 'test-vertex-credential-id',
+      type: 'oauth',
+      workspaceId: 'test-workspace',
+      accountId: 'test-vertex-credential-id',
+    },
+    member: { role: 'admin', status: 'active' },
+    hasWorkspaceAccess: true,
+    canWriteWorkspace: true,
+    isAdmin: true,
+  }),
+}))
+
+import { BlockType } from '@/executor/constants'
+import { EvaluatorBlockHandler } from '@/executor/handlers/evaluator/evaluator-handler'
+import type { ExecutionContext } from '@/executor/types'
+import { getProviderFromModel } from '@/providers/utils'
+import type { SerializedBlock } from '@/serializer/types'
+
+const mockGetProviderFromModel = getProviderFromModel as Mock
+const mockFetch = global.fetch as unknown as Mock
+
+describe('EvaluatorBlockHandler', () => {
+  let handler: EvaluatorBlockHandler
+  let mockBlock: SerializedBlock
+  let mockContext: ExecutionContext
+
+  beforeEach(() => {
+    handler = new EvaluatorBlockHandler()
+
+    mockBlock = {
+      id: 'eval-block-1',
+      metadata: { id: BlockType.EVALUATOR, name: 'Test Evaluator' },
+      position: { x: 20, y: 20 },
+      config: { tool: BlockType.EVALUATOR, params: {} },
+      inputs: {
+        content: 'string',
+        metrics: 'json',
+        model: 'string',
+        temperature: 'number',
+      }, // Using ParamType strings
+      outputs: {},
+      enabled: true,
+    }
+
+    mockContext = {
+      workflowId: 'test-workflow-id',
+      userId: 'test-user',
+      blockStates: new Map(),
+      blockLogs: [],
+      metadata: { duration: 0 },
+      environmentVariables: {},
+      decisions: { router: new Map(), condition: new Map() },
+      loopExecutions: new Map(),
+      completedLoops: new Set(),
+      executedBlocks: new Set(),
+      activeExecutionPath: new Set(),
+    }
+
+    // Reset mocks using vi
+    vi.clearAllMocks()
+
+    // Default mock implementations
+    authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValue({
+      accountId: 'test-vertex-credential-id',
+      usedCredentialTable: false,
+    })
+    authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValue({
+      accessToken: 'mock-access-token',
+      refreshed: false,
+    })
+    mockGetProviderFromModel.mockReturnValue('openai')
+
+    // Set up fetch mock to return a successful response
+    mockFetch.mockImplementation(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ score1: 5, score2: 8 }),
+            model: 'mock-model',
+            tokens: { input: 50, output: 10, total: 60 },
+            cost: 0.002,
+            timing: { total: 200 },
+          }),
+      })
+    })
+  })
+
+  it('should handle evaluator blocks', () => {
+    expect(handler.canHandle(mockBlock)).toBe(true)
+    const nonEvalBlock: SerializedBlock = { ...mockBlock, metadata: { id: 'other' } }
+    expect(handler.canHandle(nonEvalBlock)).toBe(false)
+  })
+
+  it('should execute evaluator block correctly with basic inputs', async () => {
+    const inputs = {
+      content: 'This is the content to evaluate.',
+      metrics: [
+        { name: 'score1', description: 'First score', range: { min: 0, max: 10 } },
+        { name: 'score2', description: 'Second score', range: { min: 0, max: 10 } },
+      ],
+      model: 'gpt-4o',
+      apiKey: 'test-api-key',
+      temperature: 0.1,
+    }
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect(mockGetProviderFromModel).toHaveBeenCalledWith('gpt-4o')
+    expect(mockFetch).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        method: 'POST',
+        headers: expect.any(Object),
+        body: expect.any(String),
+      })
+    )
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+    expect(requestBody).toMatchObject({
+      provider: 'openai',
+      model: 'gpt-4o',
+      systemPrompt: expect.stringContaining(inputs.content),
+      responseFormat: expect.objectContaining({
+        schema: {
+          type: 'object',
+          properties: {
+            score1: { type: 'number' },
+            score2: { type: 'number' },
+          },
+          required: ['score1', 'score2'],
+          additionalProperties: false,
+        },
+      }),
+      temperature: 0.1,
+    })
+
+    expect(result).toEqual({
+      content: 'This is the content to evaluate.',
+      model: 'mock-model',
+      tokens: { input: 50, output: 10, total: 60 },
+      cost: {
+        input: 0,
+        output: 0,
+        total: 0,
+      },
+      score1: 5,
+      score2: 8,
+    })
+  })
+
+  it('should process JSON string content correctly', async () => {
+    const contentObj = { text: 'Evaluate this JSON.', value: 42 }
+    const inputs = {
+      content: JSON.stringify(contentObj),
+      metrics: [{ name: 'clarity', description: 'Clarity score', range: { min: 1, max: 5 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ clarity: 4 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+    expect(requestBody).toMatchObject({
+      systemPrompt: expect.stringContaining(JSON.stringify(contentObj, null, 2)),
+    })
+  })
+
+  it('should process object content correctly', async () => {
+    const contentObj = { data: [1, 2, 3], status: 'ok' }
+    const inputs = {
+      content: contentObj,
+      metrics: [
+        { name: 'completeness', description: 'Data completeness', range: { min: 0, max: 1 } },
+      ],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ completeness: 1 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+    expect(requestBody).toMatchObject({
+      systemPrompt: expect.stringContaining(JSON.stringify(contentObj, null, 2)),
+    })
+  })
+
+  it('should parse valid JSON response correctly', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: '```json\n{ "quality": 9 }\n```',
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).quality).toBe(9)
+  })
+
+  it('should handle invalid/non-JSON response gracefully (scores = 0)', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 5 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: 'Sorry, I cannot provide a score.',
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).score).toBe(0)
+  })
+
+  it('should handle partially valid JSON response (extracts what it can)', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [
+        { name: 'accuracy', description: 'Acc', range: { min: 0, max: 1 } },
+        { name: 'fluency', description: 'Flu', range: { min: 0, max: 1 } },
+      ],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: '{ "accuracy": 1, "fluency": invalid }',
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+    expect((result as any).accuracy).toBe(0)
+    expect((result as any).fluency).toBe(0)
+  })
+
+  it('should extract metric scores ignoring case', async () => {
+    const inputs = {
+      content: 'Test',
+      metrics: [{ name: 'CamelCaseScore', description: 'Desc', range: { min: 0, max: 10 } }],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ camelcasescore: 7 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).camelcasescore).toBe(7)
+  })
+
+  it('should handle missing metrics in response (score = 0)', async () => {
+    const inputs = {
+      content: 'Test',
+      metrics: [
+        { name: 'presentScore', description: 'Desc1', range: { min: 0, max: 5 } },
+        { name: 'missingScore', description: 'Desc2', range: { min: 0, max: 5 } },
+      ],
+      apiKey: 'test-api-key',
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ presentScore: 4 }),
+            model: 'm',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect((result as any).presentscore).toBe(4)
+    expect((result as any).missingscore).toBe(0)
+  })
+
+  it('should handle server error responses', async () => {
+    const inputs = { content: 'Test error handling.', apiKey: 'test-api-key' }
+
+    // Override fetch mock to return an error
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: false,
+        status: 500,
+        json: () => Promise.resolve({ error: 'Server error' }),
+      })
+    })
+
+    await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow('Server error')
+  })
+
+  it('should handle Azure OpenAI models with endpoint and API version', async () => {
+    const inputs = {
+      content: 'Test content to evaluate',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      model: 'gpt-4o',
+      apiKey: 'test-azure-key',
+      azureEndpoint: 'https://test.openai.azure.com',
+      azureApiVersion: '2024-07-01-preview',
+    }
+
+    mockGetProviderFromModel.mockReturnValue('azure-openai')
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ quality: 8 }),
+            model: 'gpt-4o',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody).toMatchObject({
+      provider: 'azure-openai',
+      model: 'gpt-4o',
+      apiKey: 'test-azure-key',
+      azureEndpoint: 'https://test.openai.azure.com',
+      azureApiVersion: '2024-07-01-preview',
+    })
+  })
+
+  it('should handle Vertex AI models with OAuth credential', async () => {
+    const inputs = {
+      content: 'Test content to evaluate',
+      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      model: 'gemini-2.0-flash-exp',
+      vertexCredential: 'test-vertex-credential-id',
+      vertexProject: 'test-gcp-project',
+      vertexLocation: 'us-central1',
+    }
+
+    mockGetProviderFromModel.mockReturnValue('vertex')
+
+    // Mock the database query for Vertex credential
+    const mockDb = await import('@sim/db')
+    const mockAccount = {
+      id: 'test-vertex-credential-id',
+      accessToken: 'mock-access-token',
+      refreshToken: 'mock-refresh-token',
+      expiresAt: new Date(Date.now() + 3600000), // 1 hour from now
+    }
+    ;(mockDb.db.query as any).account = { findFirst: vi.fn() }
+    vi.spyOn(mockDb.db.query.account, 'findFirst').mockResolvedValue(mockAccount as any)
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ quality: 9 }),
+            model: 'gemini-2.0-flash-exp',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody).toMatchObject({
+      provider: 'vertex',
+      model: 'gemini-2.0-flash-exp',
+      vertexProject: 'test-gcp-project',
+      vertexLocation: 'us-central1',
+    })
+    expect(requestBody.apiKey).toBe('mock-access-token')
+  })
+
+  it('should use default model when not provided', async () => {
+    const inputs = {
+      content: 'Test content',
+      metrics: [{ name: 'score', description: 'Score', range: { min: 0, max: 10 } }],
+      apiKey: 'test-api-key',
+      // No model provided - should use default
+    }
+
+    mockFetch.mockImplementationOnce(() => {
+      return Promise.resolve({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            content: JSON.stringify({ score: 7 }),
+            model: 'claude-sonnet-5',
+            tokens: {},
+            cost: 0,
+            timing: {},
+          }),
+      })
+    })
+
+    await handler.execute(mockContext, mockBlock, inputs)
+
+    const fetchCallArgs = mockFetch.mock.calls[0]
+    const requestBody = JSON.parse(fetchCallArgs[1].body)
+
+    expect(requestBody.model).toBe('claude-sonnet-5')
+  })
+})

apps/sim/executor/handlers/evaluator/evaluator-handler.ts

@@ -0,0 +1,279 @@
+import { createLogger } from '@sim/logger'
+import type { BlockOutput } from '@/blocks/types'
+import { validateModelProvider } from '@/ee/access-control/utils/permission-check'
+import { BlockType, DEFAULTS, EVALUATOR } from '@/executor/constants'
+import type { BlockHandler, ExecutionContext } from '@/executor/types'
+import { buildAPIUrl, buildAuthHeaders, extractAPIErrorMessage } from '@/executor/utils/http'
+import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json'
+import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
+import { calculateCost, getProviderFromModel } from '@/providers/utils'
+import type { SerializedBlock } from '@/serializer/types'
+
+const logger = createLogger('EvaluatorBlockHandler')
+
+/**
+ * Handler for Evaluator blocks that assess content against criteria.
+ */
+export class EvaluatorBlockHandler implements BlockHandler {
+  canHandle(block: SerializedBlock): boolean {
+    return block.metadata?.id === BlockType.EVALUATOR
+  }
+
+  async execute(
+    ctx: ExecutionContext,
+    block: SerializedBlock,
+    inputs: Record<string, any>
+  ): Promise<BlockOutput> {
+    const evaluatorConfig = {
+      model: inputs.model || EVALUATOR.DEFAULT_MODEL,
+      apiKey: inputs.apiKey,
+      vertexProject: inputs.vertexProject,
+      vertexLocation: inputs.vertexLocation,
+      vertexCredential: inputs.vertexCredential,
+      bedrockAccessKeyId: inputs.bedrockAccessKeyId,
+      bedrockSecretKey: inputs.bedrockSecretKey,
+      bedrockRegion: inputs.bedrockRegion,
+    }
+
+    await validateModelProvider(ctx.userId, ctx.workspaceId, evaluatorConfig.model, ctx)
+
+    const providerId = getProviderFromModel(evaluatorConfig.model)
+
+    let finalApiKey: string | undefined = evaluatorConfig.apiKey
+    if (providerId === 'vertex' && evaluatorConfig.vertexCredential) {
+      finalApiKey = await resolveVertexCredential(
+        evaluatorConfig.vertexCredential,
+        ctx.userId,
+        'vertex-evaluator'
+      )
+    }
+
+    const processedContent = this.processContent(inputs.content)
+
+    let systemPromptObj: { systemPrompt: string; responseFormat: any } = {
+      systemPrompt: '',
+      responseFormat: null,
+    }
+
+    logger.info('Inputs for evaluator:', inputs)
+    let metrics: any[]
+    if (Array.isArray(inputs.metrics)) {
+      metrics = inputs.metrics
+    } else {
+      metrics = []
+    }
+    logger.info('Metrics for evaluator:', metrics)
+    const metricDescriptions = metrics
+      .filter((m: any) => m?.name && m.range)
+      .map((m: any) => `"${m.name}" (${m.range.min}-${m.range.max}): ${m.description || ''}`)
+      .join('\n')
+
+    const responseProperties: Record<string, any> = {}
+    metrics.forEach((m: any) => {
+      if (m?.name) {
+        responseProperties[m.name.toLowerCase()] = { type: 'number' }
+      } else {
+        logger.warn('Skipping invalid metric entry during response format generation:', m)
+      }
+    })
+
+    systemPromptObj = {
+      systemPrompt: `You are an evaluation agent. Analyze this content against the metrics and provide scores.
+      
+    Metrics:
+    ${metricDescriptions}
+
+    Content:
+    ${processedContent}
+
+    Return a JSON object with each metric name as a key and a numeric score as the value. No explanations, only scores.`,
+      responseFormat: {
+        name: EVALUATOR.RESPONSE_SCHEMA_NAME,
+        schema: {
+          type: 'object',
+          properties: responseProperties,
+          required: metrics.filter((m: any) => m?.name).map((m: any) => m.name.toLowerCase()),
+          additionalProperties: false,
+        },
+        strict: true,
+      },
+    }
+
+    if (!systemPromptObj.systemPrompt) {
+      systemPromptObj.systemPrompt =
+        'Evaluate the content and provide scores for each metric as JSON.'
+    }
+
+    try {
+      const url = buildAPIUrl('/api/providers', ctx.userId ? { userId: ctx.userId } : {})
+
+      const providerRequest: Record<string, any> = {
+        provider: providerId,
+        model: evaluatorConfig.model,
+        systemPrompt: systemPromptObj.systemPrompt,
+        responseFormat: systemPromptObj.responseFormat,
+        context: stringifyJSON([
+          {
+            role: 'user',
+            content:
+              'Please evaluate the content provided in the system prompt. Return ONLY a valid JSON with metric scores.',
+          },
+        ]),
+
+        temperature: EVALUATOR.DEFAULT_TEMPERATURE,
+        apiKey: finalApiKey,
+        azureEndpoint: inputs.azureEndpoint,
+        azureApiVersion: inputs.azureApiVersion,
+        vertexProject: evaluatorConfig.vertexProject,
+        vertexLocation: evaluatorConfig.vertexLocation,
+        bedrockAccessKeyId: evaluatorConfig.bedrockAccessKeyId,
+        bedrockSecretKey: evaluatorConfig.bedrockSecretKey,
+        bedrockRegion: evaluatorConfig.bedrockRegion,
+        workflowId: ctx.workflowId,
+        workspaceId: ctx.workspaceId,
+      }
+
+      const response = await fetch(url.toString(), {
+        method: 'POST',
+        headers: await buildAuthHeaders(ctx.userId),
+        body: stringifyJSON(providerRequest),
+      })
+
+      if (!response.ok) {
+        const errorMessage = await extractAPIErrorMessage(response)
+        throw new Error(errorMessage)
+      }
+
+      const result = await response.json()
+
+      const parsedContent = this.extractJSONFromResponse(result.content)
+
+      const metricScores = this.extractMetricScores(parsedContent, inputs.metrics)
+
+      const inputTokens = result.tokens?.input || result.tokens?.prompt || DEFAULTS.TOKENS.PROMPT
+      const outputTokens =
+        result.tokens?.output || result.tokens?.completion || DEFAULTS.TOKENS.COMPLETION
+
+      const costCalculation = calculateCost(result.model, inputTokens, outputTokens, false)
+
+      return {
+        content: inputs.content,
+        model: result.model,
+        tokens: {
+          input: inputTokens,
+          output: outputTokens,
+          total: result.tokens?.total || DEFAULTS.TOKENS.TOTAL,
+        },
+        cost: {
+          input: costCalculation.input,
+          output: costCalculation.output,
+          total: costCalculation.total,
+        },
+        ...metricScores,
+      }
+    } catch (error) {
+      logger.error('Evaluator execution failed:', error)
+      throw error
+    }
+  }
+
+  private processContent(content: any): string {
+    if (typeof content === 'string') {
+      if (isJSONString(content)) {
+        const parsed = parseJSON(content, null)
+        if (parsed) {
+          return stringifyJSON(parsed)
+        }
+        return content
+      }
+      return content
+    }
+
+    if (typeof content === 'object') {
+      return stringifyJSON(content)
+    }
+
+    return String(content || '')
+  }
+
+  private extractJSONFromResponse(responseContent: string): Record<string, any> {
+    try {
+      const contentStr = responseContent.trim()
+
+      const fullMatch = contentStr.match(/(\{[\s\S]*\})/)
+      if (fullMatch) {
+        return parseJSON(fullMatch[0], {})
+      }
+
+      if (contentStr.includes('{') && contentStr.includes('}')) {
+        const startIdx = contentStr.indexOf('{')
+        const endIdx = contentStr.lastIndexOf('}') + 1
+        const jsonStr = contentStr.substring(startIdx, endIdx)
+        return parseJSON(jsonStr, {})
+      }
+
+      return parseJSON(contentStr, {})
+    } catch (error) {
+      logger.error('Error parsing evaluator response:', error)
+      logger.error('Raw response content:', responseContent)
+      return {}
+    }
+  }
+
+  private extractMetricScores(
+    parsedContent: Record<string, any>,
+    metrics: any
+  ): Record<string, number> {
+    const metricScores: Record<string, number> = {}
+    let validMetrics: any[]
+    if (Array.isArray(metrics)) {
+      validMetrics = metrics
+    } else {
+      validMetrics = []
+    }
+
+    if (Object.keys(parsedContent).length === 0) {
+      validMetrics.forEach((metric: any) => {
+        if (metric?.name) {
+          metricScores[metric.name.toLowerCase()] = 0
+        }
+      })
+      return metricScores
+    }
+
+    validMetrics.forEach((metric: any) => {
+      if (!metric?.name) {
+        logger.warn('Skipping invalid metric entry:', metric)
+        return
+      }
+
+      const score = this.findMetricScore(parsedContent, metric.name)
+      metricScores[metric.name.toLowerCase()] = score
+    })
+
+    return metricScores
+  }
+
+  private findMetricScore(parsedContent: Record<string, any>, metricName: string): number {
+    const lowerMetricName = metricName.toLowerCase()
+
+    if (parsedContent[metricName] !== undefined) {
+      return Number(parsedContent[metricName])
+    }
+
+    if (parsedContent[lowerMetricName] !== undefined) {
+      return Number(parsedContent[lowerMetricName])
+    }
+
+    const matchingKey = Object.keys(parsedContent).find((key) => {
+      return typeof key === 'string' && key.toLowerCase() === lowerMetricName
+    })
+
+    if (matchingKey) {
+      return Number(parsedContent[matchingKey])
+    }
+
+    logger.warn(`Metric "${metricName}" not found in LLM response`)
+    return 0
+  }
+}