chore: import upstream snapshot with attribution

test/matchers/agent-rubric.test.ts

@@ -0,0 +1,145 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import type { ApiProvider, ProviderResponse } from '../../src/types/index';
+
+const mocks = vi.hoisted(() => {
+  const createProvider = (id: string): ApiProvider =>
+    ({
+      id: () => id,
+      config: {},
+      callApi: vi.fn(
+        async (): Promise<ProviderResponse> => ({
+          output: JSON.stringify({ pass: true, score: 1, reason: 'agent verified evidence' }),
+          metadata: { toolCalls: [{ name: 'read_file' }] },
+          tokenUsage: { total: 5, prompt: 3, completion: 2 },
+        }),
+      ),
+    }) as ApiProvider;
+
+  const codexProvider = createProvider('openai:codex-sdk');
+  const claudeProvider = createProvider('anthropic:claude-agent-sdk');
+  const textProvider = createProvider('openai:responses:gpt-5.5');
+
+  return {
+    claudeProvider,
+    codexProvider,
+    getCodexDefaultProviders: vi.fn(),
+    getDefaultProviders: vi.fn(),
+    loadApiProvider: vi.fn(),
+    textProvider,
+  };
+});
+
+vi.mock('../../src/providers/openai/codexDefaults', () => ({
+  getCodexDefaultProviders: mocks.getCodexDefaultProviders,
+}));
+
+vi.mock('../../src/providers/defaults', () => ({
+  getDefaultProviders: mocks.getDefaultProviders,
+}));
+
+vi.mock('../../src/providers/index', () => ({
+  loadApiProvider: mocks.loadApiProvider,
+}));
+
+describe('matchesAgentRubric', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    const agentResponse = async (): Promise<ProviderResponse> => ({
+      output: JSON.stringify({ pass: true, score: 1, reason: 'agent verified evidence' }),
+      metadata: { toolCalls: [{ name: 'read_file' }] },
+      tokenUsage: { total: 5, prompt: 3, completion: 2 },
+    });
+    mocks.codexProvider.callApi = vi.fn(agentResponse) as ApiProvider['callApi'];
+    mocks.claudeProvider.callApi = vi.fn(agentResponse) as ApiProvider['callApi'];
+    mocks.textProvider.callApi = vi.fn(agentResponse) as ApiProvider['callApi'];
+    mocks.getCodexDefaultProviders.mockReturnValue({
+      llmRubricProvider: mocks.codexProvider,
+    });
+    mocks.getDefaultProviders.mockResolvedValue({
+      gradingJsonProvider: mocks.codexProvider,
+      llmRubricProvider: mocks.codexProvider,
+    });
+    mocks.loadApiProvider.mockResolvedValue(mocks.claudeProvider);
+  });
+
+  it('uses the isolated Codex default provider when none is configured', async () => {
+    const { matchesAgentRubric } = await import('../../src/matchers/agent');
+
+    const result = await matchesAgentRubric('Verify the claimed file exists', 'It exists', {});
+
+    expect(mocks.getCodexDefaultProviders).toHaveBeenCalledTimes(1);
+    expect(mocks.codexProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(mocks.codexProvider.callApi).toHaveBeenCalledWith(
+      expect.stringContaining('untrusted evidence'),
+      expect.objectContaining({
+        prompt: expect.objectContaining({ label: 'agent-rubric' }),
+      }),
+    );
+    expect(result).toEqual(
+      expect.objectContaining({
+        pass: true,
+        score: 1,
+        metadata: expect.objectContaining({
+          agentProvider: 'openai:codex-sdk',
+          toolCalls: [{ name: 'read_file' }],
+        }),
+      }),
+    );
+  });
+
+  it('accepts an explicitly configured agent runtime', async () => {
+    const { matchesAgentRubric } = await import('../../src/matchers/agent');
+
+    const result = await matchesAgentRubric('Inspect the artifact', 'done', {
+      provider: 'anthropic:claude-agent-sdk',
+    });
+
+    expect(mocks.loadApiProvider).toHaveBeenCalledWith('anthropic:claude-agent-sdk', {
+      basePath: undefined,
+    });
+    expect(mocks.claudeProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(result.metadata).toEqual(
+      expect.objectContaining({ agentProvider: 'anthropic:claude-agent-sdk' }),
+    );
+  });
+
+  it('rejects an explicitly configured plain text grader instead of falling back', async () => {
+    const { matchesAgentRubric } = await import('../../src/matchers/agent');
+    mocks.loadApiProvider.mockResolvedValue(mocks.textProvider);
+
+    await expect(
+      matchesAgentRubric('Inspect the artifact', 'done', {
+        provider: 'openai:responses:gpt-5.5',
+      }),
+    ).rejects.toThrow('agent-rubric assertion requires an agentic grading provider');
+
+    expect(mocks.codexProvider.callApi).not.toHaveBeenCalled();
+    expect(mocks.textProvider.callApi).not.toHaveBeenCalled();
+  });
+
+  it('applies llm-rubric threshold semantics to agent results', async () => {
+    const { matchesAgentRubric } = await import('../../src/matchers/agent');
+    mocks.codexProvider.callApi = vi.fn(
+      async (): Promise<ProviderResponse> => ({
+        output: JSON.stringify({ pass: true, score: 0.4, reason: 'partial evidence' }),
+      }),
+    ) as ApiProvider['callApi'];
+
+    await expect(
+      matchesAgentRubric(
+        'Verify evidence',
+        'done',
+        {},
+        {},
+        { type: 'agent-rubric', value: 'Verify evidence', threshold: 0.5 },
+      ),
+    ).resolves.toEqual(
+      expect.objectContaining({
+        pass: false,
+        score: 0.4,
+        reason: 'partial evidence',
+      }),
+    );
+  });
+});

test/matchers/answer-relevance.test.ts

@@ -0,0 +1,215 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesAnswerRelevance } from '../../src/matchers/rag';
+import { ANSWER_RELEVANCY_GENERATE } from '../../src/prompts/index';
+import {
+  DefaultEmbeddingProvider,
+  DefaultGradingProvider,
+} from '../../src/providers/openai/defaults';
+
+import type { OpenAiEmbeddingProvider } from '../../src/providers/openai/embedding';
+
+describe('matchesAnswerRelevance', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockReset();
+    vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockReset();
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValue({
+      output: 'foobar',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+    vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockResolvedValue({
+      embedding: [1, 0, 0],
+      tokenUsage: { total: 5, prompt: 2, completion: 3 },
+    });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should pass when the relevance score is above the threshold', async () => {
+    const input = 'Input text';
+    const output = 'Sample output';
+    const threshold = 0.5;
+
+    const mockCallApi = vi.spyOn(DefaultGradingProvider, 'callApi');
+    mockCallApi.mockImplementation(() => {
+      return Promise.resolve({
+        output: 'foobar',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    const mockCallEmbeddingApi = vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi');
+    mockCallEmbeddingApi.mockImplementation(function (this: OpenAiEmbeddingProvider) {
+      return Promise.resolve({
+        embedding: [1, 0, 0],
+        tokenUsage: { total: 5, prompt: 2, completion: 3 },
+      });
+    });
+
+    await expect(matchesAnswerRelevance(input, output, threshold)).resolves.toEqual({
+      pass: true,
+      reason: 'Relevance 1.00 is greater than threshold 0.5',
+      score: 1,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+      metadata: {
+        generatedQuestions: expect.arrayContaining([
+          expect.objectContaining({
+            question: expect.any(String),
+            similarity: expect.any(Number),
+          }),
+        ]),
+        averageSimilarity: 1,
+        threshold: 0.5,
+      },
+    });
+    expect(mockCallApi).toHaveBeenCalledWith(
+      expect.stringContaining(ANSWER_RELEVANCY_GENERATE.slice(0, 50)),
+      expect.any(Object),
+    );
+    expect(mockCallEmbeddingApi).toHaveBeenCalledWith('Input text');
+  });
+
+  it('should fail when the relevance score is below the threshold', async () => {
+    const input = 'Input text';
+    const output = 'Different output';
+    const threshold = 0.5;
+
+    const mockCallApi = vi.spyOn(DefaultGradingProvider, 'callApi');
+    mockCallApi.mockImplementation((text) => {
+      return Promise.resolve({
+        output: text,
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    const mockCallEmbeddingApi = vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi');
+    mockCallEmbeddingApi.mockImplementation((text) => {
+      if (text.includes('Input text')) {
+        return Promise.resolve({
+          embedding: [1, 0, 0],
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        });
+      } else if (text.includes('Different output')) {
+        return Promise.resolve({
+          embedding: [0, 1, 0],
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        });
+      }
+      return Promise.reject(new Error(`Unexpected input ${text}`));
+    });
+
+    await expect(matchesAnswerRelevance(input, output, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Relevance 0.00 is less than threshold 0.5',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+      metadata: {
+        generatedQuestions: expect.arrayContaining([
+          expect.objectContaining({
+            question: expect.any(String),
+            similarity: expect.any(Number),
+          }),
+        ]),
+        averageSimilarity: 0,
+        threshold: 0.5,
+      },
+    });
+    expect(mockCallApi).toHaveBeenCalledWith(
+      expect.stringContaining(ANSWER_RELEVANCY_GENERATE.slice(0, 50)),
+      expect.any(Object),
+    );
+    expect(mockCallEmbeddingApi).toHaveBeenCalledWith(
+      expect.stringContaining(ANSWER_RELEVANCY_GENERATE.slice(0, 50)),
+    );
+  });
+
+  it('tracks token usage for successful calls', async () => {
+    const input = 'Input text';
+    const output = 'Sample output';
+    const threshold = 0.5;
+
+    const result = await matchesAnswerRelevance(input, output, threshold);
+
+    expect(result.tokensUsed?.total).toBeGreaterThan(0);
+    expect(result.tokensUsed?.prompt).toBeGreaterThan(0);
+    expect(result.tokensUsed?.completion).toBeGreaterThan(0);
+    expect(result.tokensUsed?.total).toBe(
+      (result.tokensUsed?.prompt || 0) + (result.tokensUsed?.completion || 0),
+    );
+
+    expect(result.tokensUsed?.total).toBe(50);
+    expect(result.tokensUsed?.cached).toBe(0);
+    expect(result.tokensUsed?.completionDetails).toBeDefined();
+  });
+
+  it('should return metadata with generated questions and similarities', async () => {
+    const input = 'What is the capital of France?';
+    const output = 'The capital of France is Paris.';
+    const threshold = 0.7;
+
+    // Mock 3 different generated questions
+    let callCount = 0;
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(() => {
+      const questions = [
+        'What is the capital city of France?',
+        'Which city is the capital of France?',
+        "What is France's capital?",
+      ];
+      return Promise.resolve({
+        output: questions[callCount++ % 3],
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    // Mock embeddings with varying similarities
+    vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockImplementation((text) => {
+      if (text === input) {
+        return Promise.resolve({
+          embedding: [1, 0, 0],
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        });
+      } else if (text.includes('capital') && text.includes('France')) {
+        // Similar questions get high similarity
+        return Promise.resolve({
+          embedding: [0.9, 0.1, 0],
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        });
+      }
+      return Promise.resolve({
+        embedding: [0.8, 0.2, 0],
+        tokenUsage: { total: 5, prompt: 2, completion: 3 },
+      });
+    });
+
+    const result = await matchesAnswerRelevance(input, output, threshold);
+
+    expect(result.metadata).toBeDefined();
+    expect(result.metadata?.generatedQuestions).toHaveLength(3);
+    expect(result.metadata?.generatedQuestions[0]).toMatchObject({
+      question: expect.stringContaining('capital'),
+      similarity: expect.any(Number),
+    });
+    expect(result.metadata?.averageSimilarity).toBeCloseTo(0.99, 2);
+    expect(result.metadata?.threshold).toBe(0.7);
+    expect(result.pass).toBe(true);
+  });
+});

test/matchers/callProviderWithContext.test.ts

@@ -0,0 +1,167 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { callProviderWithContext } from '../../src/matchers/providers';
+import { withProviderCallExecutionContext } from '../../src/scheduler/providerCallExecutionContext';
+import { ProviderGroupedCallQueue } from '../../src/scheduler/providerCallQueue';
+import { wrapProviderWithRateLimiting } from '../../src/scheduler/providerWrapper';
+import { createMockProvider } from '../factories/provider';
+
+import type { RateLimitRegistry } from '../../src/scheduler/rateLimitRegistry';
+import type {
+  ApiProvider,
+  ProviderResponse,
+  RateLimitRegistryRef,
+  VarValue,
+} from '../../src/types/index';
+
+function createProvider(response: ProviderResponse = { output: 'ok' }): ApiProvider {
+  return createMockProvider({ id: 'test-grader', response });
+}
+
+function createRegistry(): RateLimitRegistryRef & {
+  executeSpy: ReturnType<typeof vi.fn>;
+  disposeSpy: ReturnType<typeof vi.fn>;
+} {
+  const executeSpy = vi.fn();
+  const disposeSpy = vi.fn();
+
+  return {
+    async execute(provider, callFn, options) {
+      executeSpy(provider, callFn, options);
+      return callFn();
+    },
+    dispose() {
+      disposeSpy();
+    },
+    executeSpy,
+    disposeSpy,
+  };
+}
+
+describe('callProviderWithContext', () => {
+  const vars: Record<string, VarValue> = { question: 'What is two plus two?' };
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('calls the provider directly without scheduler execution context', async () => {
+    const response = { output: 'direct response' };
+    const provider = createProvider(response);
+
+    await expect(callProviderWithContext(provider, 'grade this', 'rubric', vars)).resolves.toBe(
+      response,
+    );
+
+    expect(provider.callApi).toHaveBeenCalledWith('grade this', {
+      prompt: { raw: 'grade this', label: 'rubric' },
+      vars,
+    });
+  });
+
+  it('uses the scheduler execution context when available', async () => {
+    const provider = createProvider();
+    const registry = createRegistry();
+
+    await withProviderCallExecutionContext({ rateLimitRegistry: registry }, () =>
+      callProviderWithContext(provider, 'grade this', 'rubric', vars),
+    );
+
+    expect(registry.executeSpy).toHaveBeenCalledWith(
+      provider,
+      expect.any(Function),
+      expect.objectContaining({
+        getHeaders: expect.any(Function),
+        isRateLimited: expect.any(Function),
+        getRetryAfter: expect.any(Function),
+      }),
+    );
+    expect(provider.callApi).toHaveBeenCalledWith('grade this', {
+      prompt: { raw: 'grade this', label: 'rubric' },
+      vars,
+    });
+  });
+
+  it('propagates abort signals from the scheduler execution context', async () => {
+    const provider = createProvider();
+    const registry = createRegistry();
+    const abortController = new AbortController();
+
+    await withProviderCallExecutionContext(
+      { abortSignal: abortController.signal, rateLimitRegistry: registry },
+      () => callProviderWithContext(provider, 'grade this', 'rubric', vars),
+    );
+
+    expect(provider.callApi).toHaveBeenCalledWith(
+      'grade this',
+      {
+        prompt: { raw: 'grade this', label: 'rubric' },
+        vars,
+      },
+      { abortSignal: abortController.signal },
+    );
+  });
+
+  it('keeps scheduler execution context scoped to its callback', async () => {
+    const provider = createProvider();
+    const registry = createRegistry();
+
+    await withProviderCallExecutionContext({ rateLimitRegistry: registry }, () =>
+      callProviderWithContext(provider, 'scheduled', 'rubric', vars),
+    );
+    await callProviderWithContext(provider, 'direct', 'rubric', vars);
+
+    expect(registry.executeSpy).toHaveBeenCalledTimes(1);
+    expect(provider.callApi).toHaveBeenCalledTimes(2);
+    expect(provider.callApi).toHaveBeenLastCalledWith('direct', {
+      prompt: { raw: 'direct', label: 'rubric' },
+      vars,
+    });
+  });
+
+  it('does not double schedule providers that are already rate-limit wrapped', async () => {
+    const provider = createProvider();
+    const wrapperRegistry = createRegistry();
+    const contextRegistry = createRegistry();
+    const wrappedProvider = wrapProviderWithRateLimiting(
+      provider,
+      wrapperRegistry as unknown as RateLimitRegistry,
+    );
+
+    await withProviderCallExecutionContext({ rateLimitRegistry: contextRegistry }, () =>
+      callProviderWithContext(wrappedProvider, 'grade this', 'rubric', vars),
+    );
+
+    expect(contextRegistry.executeSpy).not.toHaveBeenCalled();
+    expect(wrapperRegistry.executeSpy).toHaveBeenCalledTimes(1);
+    expect(provider.callApi).toHaveBeenCalledWith(
+      'grade this',
+      {
+        prompt: { raw: 'grade this', label: 'rubric' },
+        vars,
+      },
+      undefined,
+    );
+  });
+
+  it('queues provider calls when a provider call queue is available', async () => {
+    const response = { output: 'queued response' };
+    const provider = createProvider(response);
+    const providerCallQueue = new ProviderGroupedCallQueue();
+
+    const promise = withProviderCallExecutionContext({ providerCallQueue }, () =>
+      callProviderWithContext(provider, 'grade this', 'rubric', vars),
+    );
+
+    expect(provider.callApi).not.toHaveBeenCalled();
+    const group = providerCallQueue.takeNextGroup();
+    expect(group).toHaveLength(1);
+    expect(group[0].providerId).toBe('test-grader');
+
+    await providerCallQueue.run(group[0]);
+    await expect(promise).resolves.toBe(response);
+    expect(provider.callApi).toHaveBeenCalledWith('grade this', {
+      prompt: { raw: 'grade this', label: 'rubric' },
+      vars,
+    });
+  });
+});

test/matchers/classification.test.ts

@@ -0,0 +1,144 @@
+import { describe, expect, it, vi } from 'vitest';
+import { matchesClassification } from '../../src/matchers/classification';
+import { HuggingfaceTextClassificationProvider } from '../../src/providers/huggingface';
+import { createMockProvider } from '../factories/provider';
+
+import type {
+  ApiProvider,
+  GradingConfig,
+  ProviderClassificationResponse,
+  ProviderResponse,
+} from '../../src/types/index';
+
+describe('matchesClassification', () => {
+  class TestGrader implements ApiProvider {
+    async callApi(): Promise<ProviderResponse> {
+      throw new Error('Not implemented');
+    }
+
+    async callClassificationApi(): Promise<ProviderClassificationResponse> {
+      return {
+        classification: {
+          classA: 0.6,
+          classB: 0.5,
+        },
+      };
+    }
+
+    id(): string {
+      return 'TestClassificationProvider';
+    }
+  }
+
+  it('should pass when the classification score is above the threshold', async () => {
+    const expected = 'classA';
+    const output = 'Sample output';
+    const threshold = 0.5;
+
+    const grader = new TestGrader();
+    const grading: GradingConfig = {
+      provider: grader,
+    };
+
+    await expect(matchesClassification(expected, output, threshold, grading)).resolves.toEqual({
+      pass: true,
+      reason: `Classification ${expected} has score 0.60 >= ${threshold}`,
+      score: 0.6,
+    });
+  });
+
+  it('should fail when the classification score is below the threshold', async () => {
+    const expected = 'classA';
+    const output = 'Different output';
+    const threshold = 0.9;
+
+    const grader = new TestGrader();
+    const grading: GradingConfig = {
+      provider: grader,
+    };
+
+    await expect(matchesClassification(expected, output, threshold, grading)).resolves.toEqual({
+      pass: false,
+      reason: `Classification ${expected} has score 0.60 < ${threshold}`,
+      score: 0.6,
+    });
+  });
+
+  it('should pass when the maximum classification score is above the threshold with undefined expected', async () => {
+    const expected = undefined;
+    const output = 'Sample output';
+    const threshold = 0.55;
+
+    const grader = new TestGrader();
+    const grading: GradingConfig = {
+      provider: grader,
+    };
+
+    await expect(matchesClassification(expected, output, threshold, grading)).resolves.toEqual({
+      pass: true,
+      reason: `Maximum classification score 0.60 >= ${threshold}`,
+      score: 0.6,
+    });
+  });
+
+  it('should fail with a maximum-score reason when expected is undefined', async () => {
+    const output = 'Sample output';
+    const threshold = 0.9;
+
+    const grader = new TestGrader();
+    const grading: GradingConfig = {
+      provider: grader,
+    };
+
+    await expect(matchesClassification(undefined, output, threshold, grading)).resolves.toEqual({
+      pass: false,
+      reason: `Maximum classification score 0.60 < ${threshold}`,
+      score: 0.6,
+    });
+  });
+
+  it('should fail cleanly when expected is undefined and no scores are returned', async () => {
+    const grading: GradingConfig = {
+      provider: Object.assign(createMockProvider({ id: 'empty-classification-provider' }), {
+        callClassificationApi: vi.fn().mockResolvedValue({ classification: {} }),
+      }),
+    };
+
+    await expect(matchesClassification(undefined, 'Sample output', 0.5, grading)).resolves.toEqual({
+      pass: false,
+      reason: 'No classification scores returned',
+      score: 0,
+    });
+  });
+
+  it('should use the overridden classification grading config', async () => {
+    const expected = 'classA';
+    const output = 'Sample output';
+    const threshold = 0.5;
+
+    const grading: GradingConfig = {
+      provider: {
+        id: 'hf:text-classification:foobar',
+      },
+    };
+
+    const mockCallApi = vi.spyOn(
+      HuggingfaceTextClassificationProvider.prototype,
+      'callClassificationApi',
+    );
+    mockCallApi.mockImplementation(function (this: HuggingfaceTextClassificationProvider) {
+      return Promise.resolve({
+        classification: { [expected]: 0.6 },
+      });
+    });
+
+    await expect(matchesClassification(expected, output, threshold, grading)).resolves.toEqual({
+      pass: true,
+      reason: `Classification ${expected} has score 0.60 >= ${threshold}`,
+      score: 0.6,
+    });
+    expect(mockCallApi).toHaveBeenCalledWith('Sample output');
+
+    mockCallApi.mockRestore();
+  });
+});

test/matchers/closed-qa.test.ts

@@ -0,0 +1,206 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesClosedQa } from '../../src/matchers/llmGrading';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+import { createMockProvider } from '../factories/provider';
+import { mockProcessEnv } from '../util/utils';
+
+import type { GradingConfig } from '../../src/types/index';
+
+describe('matchesClosedQa', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockReset();
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValue({
+      output: 'foo \n \n bar\n Y Y \n',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should pass when the closed QA check passes', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      output: 'foo \n \n bar\n Y Y \n',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    await expect(matchesClosedQa(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason: 'The submission meets the criterion:\nfoo \n \n bar\n Y Y \n',
+      score: 1,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should fail when the closed QA check fails', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      output: 'foo bar N \n',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    await expect(matchesClosedQa(input, expected, output, grading)).resolves.toEqual({
+      pass: false,
+      reason: 'The submission does not meet the criterion:\nfoo bar N \n',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should throw an error when an error occurs', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(() => {
+      throw new Error('An error occurred');
+    });
+
+    await expect(matchesClosedQa(input, expected, output, grading)).rejects.toThrow(
+      'An error occurred',
+    );
+  });
+
+  it('should handle input, criteria, and completion that need escaping', async () => {
+    const input = 'Input "text" with \\ escape characters and \\"nested\\" escapes';
+    const expected = 'Expected "output" with \\\\ escape characters and \\"nested\\" escapes';
+    const output = 'Sample "output" with \\\\ escape characters and \\"nested\\" escapes';
+    const grading = {};
+
+    let isJson = false;
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation((prompt) => {
+      try {
+        JSON.parse(prompt);
+        isJson = true;
+      } catch {
+        isJson = false;
+      }
+      return Promise.resolve({
+        output: 'foo \n \n bar\n Y Y',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+    await expect(matchesClosedQa(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason: 'The submission meets the criterion:\nfoo \n \n bar\n Y Y',
+      score: 1,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+    expect(isJson).toBeTruthy();
+  });
+
+  it('should use Nunjucks templating when PROMPTFOO_DISABLE_TEMPLATING is set', async () => {
+    const restoreEnv = mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    try {
+      const input = 'Input {{ var }}';
+      const expected = 'Expected {{ var }}';
+      const output = 'Output {{ var }}';
+      const grading: GradingConfig = {
+        provider: DefaultGradingProvider,
+      };
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValue({
+        output: 'Y',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+
+      await matchesClosedQa(input, expected, output, grading);
+
+      expect(DefaultGradingProvider.callApi).toHaveBeenCalledWith(
+        expect.stringContaining('Input {{ var }}'),
+        expect.any(Object),
+      );
+      expect(DefaultGradingProvider.callApi).toHaveBeenCalledWith(
+        expect.stringContaining('Expected {{ var }}'),
+        expect.any(Object),
+      );
+      expect(DefaultGradingProvider.callApi).toHaveBeenCalledWith(
+        expect.stringContaining('Output {{ var }}'),
+        expect.any(Object),
+      );
+    } finally {
+      restoreEnv();
+    }
+  });
+
+  it('should correctly substitute variables in custom rubricPrompt', async () => {
+    const input = 'What is the largest ocean?';
+    const expected = 'Pacific Ocean';
+    const output = 'The largest ocean is the Pacific Ocean.';
+
+    const customPrompt = `Compare these answers:
+Question: {{input}}
+Criteria: {{criteria}}
+Answer: {{completion}}
+
+Does the answer meet the criteria? Answer Y or N.`;
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: 'Y',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    const grading = {
+      rubricPrompt: customPrompt,
+      provider: createMockProvider({ callApi: mockCallApi }),
+    };
+
+    const result = await matchesClosedQa(input, expected, output, grading);
+
+    expect(result).toEqual({
+      pass: true,
+      reason: expect.any(String),
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+
+    // Verify all variables were substituted in the prompt
+    expect(mockCallApi).toHaveBeenCalledTimes(1);
+    const actualPrompt = mockCallApi.mock.calls[0][0];
+    expect(actualPrompt).toContain('Question: What is the largest ocean?');
+    expect(actualPrompt).toContain('Criteria: Pacific Ocean');
+    expect(actualPrompt).toContain('Answer: The largest ocean is the Pacific Ocean.');
+    expect(actualPrompt).not.toContain('{{input}}');
+    expect(actualPrompt).not.toContain('{{criteria}}');
+    expect(actualPrompt).not.toContain('{{completion}}');
+  });
+});

test/matchers/comparison.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest';
+import { matchesSelectBest } from '../../src/matchers/comparison';
+import { createMockProvider } from '../factories/provider';
+
+import type { ApiProvider, GradingConfig } from '../../src/types/index';
+
+function createSelectBestProvider(output: string): ApiProvider {
+  return createMockProvider({
+    id: 'select-best-test-provider',
+    response: {
+      output,
+      tokenUsage: { total: 7, prompt: 3, completion: 4 },
+    },
+  });
+}
+
+describe('matchesSelectBest', () => {
+  it('should parse multi-digit verdict indexes', async () => {
+    const provider = createSelectBestProvider('10');
+    const outputs = Array.from({ length: 12 }, (_value, index) => `Output ${index}`);
+    const grading: GradingConfig = { provider };
+
+    const result = await matchesSelectBest('choose the best output', outputs, grading);
+
+    expect(result[10]).toMatchObject({
+      pass: true,
+      score: 1,
+      reason: 'Output selected as the best: choose the best output',
+    });
+    expect(result.filter((item) => item.pass)).toHaveLength(1);
+  });
+
+  it('should return independent failure results for invalid verdicts', async () => {
+    const provider = createSelectBestProvider('no verdict');
+    const grading: GradingConfig = { provider };
+
+    const result = await matchesSelectBest('choose the best output', ['A', 'B'], grading);
+
+    expect(result).toHaveLength(2);
+    expect(result[0]).not.toBe(result[1]);
+    expect(result[0]).toMatchObject({
+      pass: false,
+      reason: 'Invalid select-best verdict: NaN',
+      tokensUsed: {
+        total: 7,
+        prompt: 3,
+        completion: 4,
+      },
+    });
+  });
+});

test/matchers/context-faithfulness.test.ts

@@ -0,0 +1,386 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesContextFaithfulness } from '../../src/matchers/rag';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+
+describe('matchesContextFaithfulness', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockReset();
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3\n',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Final verdict for each statement in order: Yes. No. Yes.',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should pass when the faithfulness score is above the threshold', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.5;
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3\n',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Final verdict for each statement in order: Yes. No. Yes.',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await expect(matchesContextFaithfulness(query, output, context, threshold)).resolves.toEqual({
+      pass: true,
+      reason: 'Faithfulness 0.67 is >= 0.5',
+      score: expect.closeTo(0.67, 0.01),
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should fail when the faithfulness score is below the threshold', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.7;
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Final verdict for each statement in order: Yes. Yes. No.',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await expect(matchesContextFaithfulness(query, output, context, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Faithfulness 0.67 is < 0.7',
+      score: expect.closeTo(0.67, 0.01),
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('tracks token usage for multiple API calls', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.5;
+
+    const result = await matchesContextFaithfulness(query, output, context, threshold);
+
+    expect(result.tokensUsed).toEqual({
+      total: 20,
+      prompt: 10,
+      completion: 10,
+      cached: 0,
+      completionDetails: expect.any(Object),
+      numRequests: 0,
+    });
+  });
+
+  it('should keep reserved faithfulness vars ahead of user vars', async () => {
+    const mockCallApi = vi
+      .fn()
+      .mockResolvedValueOnce({
+        output: 'Statement from answer.',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      })
+      .mockResolvedValueOnce({
+        output: 'Final verdict for each statement in order: Yes.',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await matchesContextFaithfulness(
+      'question from assertion',
+      'answer from provider',
+      'context from contextTransform',
+      0,
+      {
+        rubricPrompt: [
+          'question={{ question }}\nanswer={{ answer }}\nextra={{ extra }}',
+          'context={{ context }}\nstatements={{ statements }}\nextra={{ extra }}',
+        ],
+      },
+      {
+        question: 'vars question sentinel',
+        answer: 'vars answer sentinel',
+        context: 'vars context sentinel',
+        statements: 'vars statements sentinel',
+        extra: 'kept user var',
+      },
+    );
+
+    const [longformPrompt, longformCallApiContext] = mockCallApi.mock.calls[0];
+    const [nliPrompt, nliCallApiContext] = mockCallApi.mock.calls[1];
+
+    expect(longformPrompt).toContain('question=question from assertion');
+    expect(longformPrompt).toContain('answer=answer from provider');
+    expect(longformPrompt).toContain('extra=kept user var');
+    expect(longformPrompt).not.toContain('vars question sentinel');
+    expect(longformPrompt).not.toContain('vars answer sentinel');
+    expect(longformCallApiContext.vars).toMatchObject({
+      question: 'question from assertion',
+      answer: 'answer from provider',
+      extra: 'kept user var',
+    });
+
+    expect(nliPrompt).toContain('context=context from contextTransform');
+    expect(nliPrompt).toContain('statements=Statement from answer.');
+    expect(nliPrompt).toContain('extra=kept user var');
+    expect(nliPrompt).not.toContain('vars context sentinel');
+    expect(nliPrompt).not.toContain('vars statements sentinel');
+    expect(nliCallApiContext.vars).toMatchObject({
+      context: 'context from contextTransform',
+      statements: ['Statement from answer.'],
+      extra: 'kept user var',
+    });
+  });
+
+  it('should fail when no verdict markers are returned', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.5;
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output:
+            'I apologize, but I cannot create statements or provide an analysis based on the given context.',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await expect(matchesContextFaithfulness(query, output, context, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Faithfulness 0.00 is < 0.5',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should count missing final-answer verdicts as unsupported', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.5;
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Final verdict for each statement in order: Yes.',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await expect(matchesContextFaithfulness(query, output, context, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Faithfulness 0.33 is < 0.5',
+      score: expect.closeTo(0.33, 0.01),
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should count missing line-by-line verdicts as unsupported', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.5;
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nverdict: yes',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await expect(matchesContextFaithfulness(query, output, context, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Faithfulness 0.33 is < 0.5',
+      score: expect.closeTo(0.33, 0.01),
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should clamp score when verdict count exceeds statement count', async () => {
+    const query = 'Query text';
+    const output = 'Output text';
+    const context = 'Context text';
+    const threshold = 0.5;
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'Statement 1\nStatement 2\nStatement 3',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      })
+      .mockImplementationOnce(() => {
+        return Promise.resolve({
+          output: 'verdict: no\nverdict: no\nverdict: no\nverdict: no\nverdict: no',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+    const callApiSpy = vi.spyOn(DefaultGradingProvider, 'callApi');
+    callApiSpy.mockReset();
+    callApiSpy.mockImplementation(mockCallApi);
+
+    await expect(matchesContextFaithfulness(query, output, context, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Faithfulness 0.00 is < 0.5',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  describe('Array Context Support', () => {
+    it('should handle array of context chunks', async () => {
+      const query = 'What is the capital of France?';
+      const output = 'Paris is the capital of France.';
+      const contextChunks = [
+        'Paris is the capital and largest city of France.',
+        'France is located in Western Europe.',
+        'The country has a rich cultural heritage.',
+      ];
+      const threshold = 0.5;
+
+      const result = await matchesContextFaithfulness(query, output, contextChunks, threshold);
+
+      // Should successfully process array context without errors
+      expect(result.score).toBeGreaterThanOrEqual(0);
+      expect(result.score).toBeLessThanOrEqual(1);
+      expect(typeof result.pass).toBe('boolean');
+      expect(result.reason).toBeDefined();
+    });
+
+    it('should handle single string context (backward compatibility)', async () => {
+      const query = 'What is the capital of France?';
+      const output = 'Paris is the capital of France.';
+      const context = 'Paris is the capital and largest city of France.';
+      const threshold = 0.5;
+
+      const result = await matchesContextFaithfulness(query, output, context, threshold);
+
+      // Should successfully process string context without errors
+      expect(result.score).toBeGreaterThanOrEqual(0);
+      expect(result.score).toBeLessThanOrEqual(1);
+      expect(typeof result.pass).toBe('boolean');
+      expect(result.reason).toBeDefined();
+    });
+  });
+});

test/matchers/context-recall.test.ts

@@ -0,0 +1,369 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesContextRecall } from '../../src/matchers/rag';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+
+describe('matchesContextRecall', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockReset();
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValue({
+      output: 'foo [Attributed]\nbar [Not attributed]\nbaz [Attributed]\n',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should pass when the recall score is above the threshold', async () => {
+    const context = 'Context text';
+    const groundTruth = 'Ground truth text';
+    const threshold = 0.5;
+
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output: 'foo [Attributed]\nbar [Not attributed]\nbaz [Attributed]\n',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesContextRecall(context, groundTruth, threshold)).resolves.toEqual({
+      pass: true,
+      reason: 'Recall 0.67 is >= 0.5',
+      score: expect.closeTo(0.67, 0.01),
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+      metadata: {
+        sentenceAttributions: expect.arrayContaining([
+          expect.objectContaining({
+            sentence: expect.any(String),
+            attributed: expect.any(Boolean),
+          }),
+        ]),
+        totalSentences: 3,
+        attributedSentences: 2,
+        score: expect.closeTo(0.67, 0.01),
+      },
+    });
+  });
+
+  it('should fail when the recall score is below the threshold', async () => {
+    const context = 'Context text';
+    const groundTruth = 'Ground truth text';
+    const threshold = 0.9;
+
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output: 'foo [Attributed]\nbar [Not attributed]\nbaz [Attributed]',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesContextRecall(context, groundTruth, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Recall 0.67 is < 0.9',
+      score: expect.closeTo(0.67, 0.01),
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+      metadata: {
+        sentenceAttributions: expect.arrayContaining([
+          expect.objectContaining({
+            sentence: expect.any(String),
+            attributed: expect.any(Boolean),
+          }),
+        ]),
+        totalSentences: 3,
+        attributedSentences: 2,
+        score: expect.closeTo(0.67, 0.01),
+      },
+    });
+  });
+
+  it('should return detailed metadata with sentence attributions', async () => {
+    const context = 'The capital of France is Paris. It has the Eiffel Tower.';
+    const groundTruth =
+      'Paris is the capital of France. The Eiffel Tower is located there. London is the capital of UK.';
+    const threshold = 0.6;
+
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output:
+          'Paris is the capital of France. [Attributed]\nThe Eiffel Tower is located there. [Attributed]\nLondon is the capital of UK. [Not attributed]',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRecall(context, groundTruth, threshold);
+
+    expect(result.metadata).toBeDefined();
+    expect(result.metadata?.sentenceAttributions).toHaveLength(3);
+    expect(result.metadata?.sentenceAttributions[0]).toEqual({
+      sentence: 'Paris is the capital of France',
+      attributed: true,
+    });
+    expect(result.metadata?.sentenceAttributions[1]).toEqual({
+      sentence: 'The Eiffel Tower is located there',
+      attributed: true,
+    });
+    expect(result.metadata?.sentenceAttributions[2]).toEqual({
+      sentence: 'London is the capital of UK',
+      attributed: false,
+    });
+    expect(result.metadata?.totalSentences).toBe(3);
+    expect(result.metadata?.attributedSentences).toBe(2);
+    expect(result.metadata?.score).toBeCloseTo(0.67, 2);
+    expect(result.pass).toBe(true);
+  });
+
+  it('should keep reserved context and ground truth vars ahead of user vars', async () => {
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '1. Statement from the expected answer. [Attributed]',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await matchesContextRecall(
+      'context from contextTransform',
+      'ground truth from assertion',
+      0,
+      {
+        rubricPrompt: 'context={{ context }}\ngroundTruth={{ groundTruth }}\nextra={{ extra }}',
+      },
+      {
+        context: 'vars context sentinel',
+        groundTruth: 'vars ground truth sentinel',
+        extra: 'kept user var',
+      },
+    );
+
+    const [prompt, callApiContext] = mockCallApi.mock.calls[0];
+
+    expect(prompt).toContain('context=context from contextTransform');
+    expect(prompt).toContain('groundTruth=ground truth from assertion');
+    expect(prompt).toContain('extra=kept user var');
+    expect(prompt).not.toContain('vars context sentinel');
+    expect(prompt).not.toContain('vars ground truth sentinel');
+    expect(callApiContext.vars).toMatchObject({
+      context: 'context from contextTransform',
+      groundTruth: 'ground truth from assertion',
+      extra: 'kept user var',
+    });
+  });
+
+  describe('Preamble Filtering (Issue #1506)', () => {
+    it('should ignore preamble text and only count classification lines', async () => {
+      const context = 'The service owner must sign the DR plan.';
+      const groundTruth = 'The signature from the service owner on the DR plan is requested.';
+      const threshold = 0.9;
+
+      // Simulate LLM output with preamble text before the classification
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'Let me analyze each sentence in the answer.\n\n' +
+            '1. The signature from the service owner on the DR plan is requested. The context clearly states this requirement. [Attributed]',
+          tokenUsage: { total: 20, prompt: 10, completion: 10 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(context, groundTruth, threshold);
+
+      // Should be 1.0 (1/1 attributed) not 0.5 (1/2) due to preamble being ignored
+      expect(result.pass).toBe(true);
+      expect(result.score).toBe(1.0);
+      expect(result.metadata?.totalSentences).toBe(1);
+      expect(result.metadata?.attributedSentences).toBe(1);
+    });
+
+    it('should handle multi-line preamble with explanation text', async () => {
+      const context = 'Albert Einstein was born in Germany.';
+      const groundTruth = 'Einstein was German. He won the Nobel Prize.';
+      const threshold = 0.4;
+
+      // Simulate verbose LLM output with multiple preamble lines
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            "I'll analyze each sentence to determine if it can be attributed to the context.\n" +
+            '\n' +
+            'Here is my analysis:\n' +
+            '\n' +
+            '1. Einstein was German. This can be inferred from the context. [Attributed]\n' +
+            '2. He won the Nobel Prize. There is no mention of this in the context. [Not Attributed]',
+          tokenUsage: { total: 30, prompt: 15, completion: 15 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(context, groundTruth, threshold);
+
+      // Should be 0.5 (1/2) not lower due to preamble lines being counted
+      expect(result.score).toBe(0.5);
+      expect(result.metadata?.totalSentences).toBe(2);
+      expect(result.metadata?.attributedSentences).toBe(1);
+      expect(result.pass).toBe(true);
+    });
+
+    it('should handle case-insensitive attribution markers', async () => {
+      const context = 'Test context';
+      const groundTruth = 'Test ground truth';
+      const threshold = 0.5;
+
+      // Test various case combinations
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            '1. First sentence [ATTRIBUTED]\n' +
+            '2. Second sentence [not attributed]\n' +
+            '3. Third sentence [Attributed]',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(context, groundTruth, threshold);
+
+      // All 3 lines should be recognized as classification lines
+      expect(result.metadata?.totalSentences).toBe(3);
+      // Case-insensitive matching: [ATTRIBUTED] and [Attributed] both count as attributed
+      // Line 1: [ATTRIBUTED] - counted as attributed
+      // Line 2: [not attributed] - NOT counted as attributed (contains "not")
+      // Line 3: [Attributed] - counted as attributed
+      expect(result.metadata?.attributedSentences).toBe(2);
+    });
+
+    it('should let not-attributed markers take precedence over attributed markers', async () => {
+      const context = 'The context supports only one sentence.';
+      const groundTruth = 'First sentence. Second sentence.';
+      const threshold = 0.5;
+
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'First sentence. [Attributed]\n' + 'Second sentence. [Not Attributed] [Attributed]',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(context, groundTruth, threshold);
+
+      expect(result.score).toBe(0.5);
+      expect(result.metadata?.attributedSentences).toBe(1);
+      expect(result.metadata?.sentenceAttributions).toEqual([
+        { sentence: 'First sentence', attributed: true },
+        { sentence: 'Second sentence', attributed: false },
+      ]);
+    });
+
+    it('should return score 0 when LLM returns no classification lines', async () => {
+      const context = 'Test context';
+      const groundTruth = 'Test ground truth';
+      const threshold = 0.5;
+
+      // LLM returns only explanation without any classification markers
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'I cannot determine if the statements are attributed to the context.\n' +
+            'The context does not provide enough information.',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(context, groundTruth, threshold);
+
+      expect(result.score).toBe(0);
+      expect(result.metadata?.totalSentences).toBe(0);
+      expect(result.pass).toBe(false);
+    });
+  });
+
+  describe('Array Context Support', () => {
+    it('should handle array of context chunks', async () => {
+      const contextChunks = [
+        'Paris is the capital of France and has many landmarks.',
+        'The Eiffel Tower is a famous iron lattice tower in Paris.',
+        'France is located in Western Europe.',
+      ];
+      const groundTruth = 'Paris is the capital of France. The Eiffel Tower is located there.';
+      const threshold = 0.5;
+
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'Paris is the capital of France [Attributed]\nThe Eiffel Tower is located there [Attributed]',
+          tokenUsage: { total: 15, prompt: 8, completion: 7 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(contextChunks, groundTruth, threshold);
+
+      expect(result.pass).toBe(true);
+      expect(result.score).toBe(1.0); // 2/2 sentences attributed
+      expect(result.reason).toContain('Recall 1.00 is >= 0.5');
+
+      // Verify that context chunks were joined and included in the LLM prompt
+      const callArgs = mockCallApi.mock.calls[0];
+      const prompt = callArgs[0];
+      expect(prompt).toContain(contextChunks.join('\n\n'));
+    });
+
+    it('should handle single string context (backward compatibility)', async () => {
+      const context = 'Paris is the capital of France. The Eiffel Tower is located there.';
+      const groundTruth = 'Paris is the capital of France. The Eiffel Tower is located there.';
+      const threshold = 0.8;
+
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'Paris is the capital of France [Attributed]\nThe Eiffel Tower is located there [Attributed]',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRecall(context, groundTruth, threshold);
+
+      expect(result.pass).toBe(true);
+      expect(result.score).toBe(1.0);
+
+      // Should work exactly like before with string context
+      const callArgs = mockCallApi.mock.calls[0];
+      const prompt = callArgs[0];
+      expect(prompt).toContain(context);
+    });
+  });
+});

test/matchers/context-relevance.test.ts

@@ -0,0 +1,441 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesContextRelevance } from '../../src/matchers/rag';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+
+describe('matchesContextRelevance (RAGAS Context Relevance)', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockReset();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should calculate relevance using line-based sentence splitting', async () => {
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France.\nFrance is in Europe.\nThe weather is nice today.';
+    const threshold = 0.3;
+
+    // Mock LLM extracting 1 relevant sentence
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output: 'Paris is the capital of France',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // Context has 3 lines, LLM extracts 1 → score = 1/3 ≈ 0.33
+    expect(result.score).toBeCloseTo(0.33, 2);
+    expect(result.pass).toBe(true); // 0.33 >= 0.3
+    expect(result.reason).toContain('Context relevance');
+    expect(result.metadata?.graderOutputs).toEqual({
+      final: 'Paris is the capital of France',
+    });
+    expect(result.metadata?.extractedSentences).toEqual(['Paris is the capital of France']);
+    expect(result.metadata?.totalContextUnits).toBe(3);
+    expect(result.metadata?.relevantSentenceCount).toBe(1);
+  });
+
+  it('should segment a single-paragraph prose context into sentences', async () => {
+    // Regression: a retrieved context is usually one prose paragraph with no
+    // newlines. Splitting it on newlines alone yields a single context unit, so
+    // the denominator is 1 and the score is forced to ~1.0 regardless of how
+    // little of the context is actually relevant. Segmenting on sentence
+    // boundaries restores a meaningful denominator.
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France. France is in Europe. The weather is nice today.';
+    const threshold = 0.3;
+
+    // Mock LLM extracting 1 relevant sentence out of the 3 in the context.
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output: 'Paris is the capital of France.',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // 3 sentences in the context, 1 relevant → score = 1/3 ≈ 0.33.
+    // Before the fix this returned 1.0 (the whole paragraph counted as one unit).
+    expect(result.score).toBeCloseTo(0.33, 2);
+    expect(result.pass).toBe(true); // 0.33 >= 0.3
+    expect(result.metadata?.totalContextUnits).toBe(3);
+    expect(result.metadata?.relevantSentenceCount).toBe(1);
+  });
+
+  it('should count multiple relevant sentences in a prose grader response (single-line context)', async () => {
+    // For a single-paragraph prose context the grader echoes the relevant sentences
+    // verbatim as continuous prose (no newlines). The numerator must be segmented
+    // the same way as the denominator (by sentence here), otherwise a multi-sentence
+    // answer counts as a single relevant unit and undercounts relevance — e.g.
+    // echoing two of three sentences scored 1/3 instead of 2/3.
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France. France is in Europe. The weather is nice today.';
+    const threshold = 0.5;
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: 'Paris is the capital of France. France is in Europe.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // 2 relevant sentences out of 3 → 2/3 ≈ 0.67 (was 1/3 ≈ 0.33 when the prose
+    // answer was counted as a single line).
+    expect(result.score).toBeCloseTo(0.67, 2);
+    expect(result.metadata?.totalContextUnits).toBe(3);
+    expect(result.metadata?.relevantSentenceCount).toBe(2);
+  });
+
+  it('should still segment prose when the context carries an incidental trailing newline', async () => {
+    // Regression: a context loaded from a file/template/YAML block scalar almost
+    // always carries a trailing newline. Keying segmentation off the mere presence
+    // of a newline collapsed such a context back to a single unit, reviving the
+    // forced ~1.0 score. The mode must key off two-or-more non-empty lines instead.
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France. France is in Europe. The weather is nice today.\n';
+    const threshold = 0.3;
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: 'Paris is the capital of France.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // Still 3 sentences despite the trailing newline → 1/3 ≈ 0.33 (not 1.0).
+    expect(result.score).toBeCloseTo(0.33, 2);
+    expect(result.metadata?.totalContextUnits).toBe(3);
+    expect(result.metadata?.relevantSentenceCount).toBe(1);
+  });
+
+  it('should not inflate the score when the grader returns a numbered list', async () => {
+    // Regression: the grading prompt does not constrain output format, and an LLM
+    // commonly returns the relevant sentences as a numbered list. Sentence-splitting
+    // the grader output would count each "1."/"2." marker as its own unit, inflating
+    // the numerator. The grader output must be counted by line instead.
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France. France is in Europe. The weather is nice today.';
+    const threshold = 0.5;
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '1. Paris is the capital of France.\n2. France is in Europe.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // 2 relevant lines out of 3 context sentences → 2/3 ≈ 0.67 (not 1.0).
+    expect(result.score).toBeCloseTo(0.67, 2);
+    expect(result.metadata?.totalContextUnits).toBe(3);
+    expect(result.metadata?.relevantSentenceCount).toBe(2);
+  });
+
+  it('should not inflate the score for an inline (single-line) numbered list grader output', async () => {
+    // Regression: when the grader returns the numbered list on ONE line (no newlines),
+    // sentence-splitting strands each "1."/"2." marker as its own segment. Counting
+    // those markers inflated the numerator to 4, capping the score at 1.0. The bare
+    // markers must be dropped so only the two real sentences count.
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France. France is in Europe. The weather is nice today.';
+    const threshold = 0.5;
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '1. Paris is the capital of France. 2. France is in Europe.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // 2 relevant sentences out of 3 → 2/3 ≈ 0.67 (was 3/3 = 1.0 when the stranded
+    // "1." and "2." markers were counted as relevant units).
+    expect(result.score).toBeCloseTo(0.67, 2);
+    expect(result.metadata?.totalContextUnits).toBe(3);
+    expect(result.metadata?.relevantSentenceCount).toBe(2);
+  });
+
+  it('should count the grader output by line, not sentence, against a multi-line context', async () => {
+    // Regression: for a multi-line (pre-segmented) string context, a prose grader
+    // response (no newlines) was sentence-split while the context was line-split.
+    // The mismatched units inflated the numerator and forced the score to 1.0. The
+    // grader output is now counted by non-empty line, matching the context units.
+    const input = 'What is the capital of France?';
+    const context =
+      'Paris is the capital of France. France is in Europe.\nBerlin is the capital of Germany. Germany is in Europe.';
+    const threshold = 0.5;
+
+    // Grader returns one line verbatim (prose with two sentences, no newline).
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: 'Paris is the capital of France. France is in Europe.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // Context is 2 lines; the grader's single line is 1 relevant unit → 1/2 = 0.5
+    // (was 1.0 before the fix, when the grader's two sentences were counted).
+    expect(result.score).toBe(0.5);
+    expect(result.metadata?.totalContextUnits).toBe(2);
+    expect(result.metadata?.relevantSentenceCount).toBe(1);
+  });
+
+  it('should handle insufficient information responses', async () => {
+    const input = 'What is quantum computing?';
+    const context =
+      'Paris is the capital of France. France is in Europe. The weather is nice today.';
+    const threshold = 0.5;
+
+    // Mock LLM returning insufficient information
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output: 'Insufficient Information',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    expect(result.score).toBe(0);
+    expect(result.pass).toBe(false);
+    expect(result.metadata?.insufficientInformation).toBe(true);
+    expect(result.metadata?.graderOutputs).toEqual({
+      final: 'Insufficient Information',
+    });
+    expect(result.metadata?.extractedSentences).toEqual([]);
+    expect(result.metadata?.relevantSentenceCount).toBe(0);
+  });
+
+  it('should handle multiple extracted sentences', async () => {
+    const input = 'Tell me about France and Germany';
+    const context =
+      'Paris is the capital of France.\nBerlin is the capital of Germany.\nThe weather is nice.\nFrance and Germany are in Europe.';
+    const threshold = 0.5;
+
+    // Mock LLM extracting 3 relevant sentences (with newlines)
+    const mockCallApi = vi.fn().mockImplementation(() => {
+      return Promise.resolve({
+        output:
+          'Paris is the capital of France.\nBerlin is the capital of Germany.\nFrance and Germany are in Europe.',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    const result = await matchesContextRelevance(input, context, threshold);
+
+    // Context has 4 lines, LLM extracts 3 → score = 3/4 = 0.75
+    expect(result.score).toBe(0.75);
+    expect(result.pass).toBe(true);
+    expect(result.metadata?.totalContextUnits).toBe(4);
+    expect(result.metadata?.relevantSentenceCount).toBe(3);
+    expect(result.metadata?.extractedSentences).toHaveLength(3);
+  });
+
+  describe('Formatted Document Test', () => {
+    it('should handle formatted documents better than naive line splitting', async () => {
+      const query = 'Who does the leave policy apply to?';
+      const formattedPolicyContext = `Company Leave Policy & Guidelines
+
+HR System **May 2024**
+
+Scope and Applicability
+The revised leave policy is applicable from 1st May 2024. The scope of this policy covers
+entitlement and guidelines for all categories of leaves in the office. This policy will
+include people transferred into the office under all incoming structured programs, short-
+term/permanent transfers
+
+All permanent employees i.e. Engineering Team, Business Services Team and includes all employees who go out of office on Cross Office Staffing
+
+This policy will include people transferred into the office under all incoming structured programs, short term / permanent transfers
+
+This policy excludes all staff going on any outgoing structured programs, short term/ permanent transfers, people going out of office under any of the other global mobility programs`;
+
+      // Mock LLM extracting multiple relevant lines (formatted with line breaks)
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'All permanent employees i.e. Engineering Team, Business Services Team\nThis policy will include people transferred into the office under all incoming structured programs\nThe scope of this policy covers entitlement and guidelines for all categories of leaves in the office',
+          tokenUsage: { total: 15, prompt: 10, completion: 5 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, formattedPolicyContext, 0.1);
+
+      // Formatted policy context has many lines, demonstrates improvement from chunk-based approach
+      expect(result.score).toBeGreaterThan(0.05); // Much better than original 0.05
+      expect(result.pass).toBe(true);
+      expect(result.metadata?.totalContextUnits).toBeGreaterThan(5); // Many lines in policy
+      expect(result.metadata?.relevantSentenceCount).toBe(3);
+
+      // Still better than original 0.05, but chunk-based approach is even better
+    });
+  });
+
+  describe('Chunk-Based Context (New Feature)', () => {
+    it('should handle array of context chunks', async () => {
+      const query = 'What are the benefits of RAG systems?';
+      const contextChunks = [
+        'RAG systems improve factual accuracy by incorporating external knowledge sources.',
+        'They reduce hallucinations in large language models through grounded responses.',
+        'RAG enables up-to-date information retrieval beyond training data cutoffs.',
+        'The weather forecast shows rain this weekend.', // Irrelevant chunk
+      ];
+      const threshold = 0.5;
+
+      // Mock LLM extracting 3 relevant chunks out of 4 total (with line breaks)
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output:
+            'RAG systems improve factual accuracy by incorporating external knowledge sources.\nThey reduce hallucinations in large language models through grounded responses.\nRAG enables up-to-date information retrieval beyond training data cutoffs.',
+          tokenUsage: { total: 20, prompt: 10, completion: 10 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, contextChunks, threshold);
+
+      // With 4 chunks total and 3 relevant = 3/4 = 0.75
+      expect(result.score).toBe(0.75);
+      expect(result.pass).toBe(true);
+      expect(result.metadata?.totalContextUnits).toBe(4);
+      expect(result.metadata?.contextUnits).toEqual(contextChunks);
+      expect(result.metadata?.relevantSentenceCount).toBe(3);
+    });
+
+    it('should handle single string context (backward compatibility)', async () => {
+      const query = 'What is the capital of France?';
+      const context =
+        'Paris is the capital of France.\nFrance is in Europe.\nThe weather is nice today.';
+      const threshold = 0.3;
+
+      // Mock LLM extracting 1 relevant sentence
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output: 'Paris is the capital of France',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, context, threshold);
+
+      // Should work exactly like before with line-based splitting
+      expect(result.score).toBeCloseTo(0.33, 2);
+      expect(result.pass).toBe(true);
+      expect(result.metadata?.totalContextUnits).toBe(3); // 3 lines
+      expect(result.metadata?.relevantSentenceCount).toBe(1);
+    });
+  });
+
+  describe('Edge Cases', () => {
+    it('should dedupe repeated relevant sentences and keep scores at most 1', async () => {
+      const query = 'What is the answer?';
+      const context = 'The answer is 42.';
+
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output: 'The answer is 42.\nThe answer is 42.\nThe answer is 42.',
+          tokenUsage: { total: 5, prompt: 3, completion: 2 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, context, 0.5);
+
+      expect(result.score).toBe(1);
+      expect(result.pass).toBe(true);
+      expect(result.metadata?.relevantSentenceCount).toBe(1);
+      expect(result.metadata?.extractedSentences).toEqual(['The answer is 42.']);
+    });
+
+    it('should handle empty context', async () => {
+      const query = 'What is the answer?';
+      const context = '';
+
+      // Mock response for empty context
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output: 'Insufficient Information',
+          tokenUsage: { total: 5, prompt: 3, completion: 2 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, context, 0.5);
+
+      expect(result.score).toBe(0);
+      expect(result.pass).toBe(false);
+      expect(result.metadata?.totalContextSentences).toBe(0);
+      expect(result.metadata?.insufficientInformation).toBe(true);
+    });
+
+    it('should handle very short context', async () => {
+      const query = 'What is the answer?';
+      const context = 'The answer is 42.';
+
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output: 'The answer is 42',
+          tokenUsage: { total: 5, prompt: 3, completion: 2 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, context, 0.5);
+
+      expect(result.score).toBe(1.0); // 1 extracted / 1 total = 1.0
+      expect(result.pass).toBe(true);
+    });
+
+    it('should handle malformed sentences gracefully', async () => {
+      const query = 'Test query';
+      const context = 'Fragment without end Proper sentence. Another fragment';
+
+      const mockCallApi = vi.fn().mockImplementation(() => {
+        return Promise.resolve({
+          output: 'Proper sentence',
+          tokenUsage: { total: 5, prompt: 3, completion: 2 },
+        });
+      });
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+      const result = await matchesContextRelevance(query, context, 0.5);
+
+      // Should still work with semantic sentence splitting
+      expect(result.score).toBeGreaterThan(0);
+      expect(result.metadata?.totalContextSentences).toBeGreaterThan(0);
+      expect(result.metadata?.relevantSentenceCount).toBe(1);
+    });
+  });
+});

test/matchers/factuality.test.ts

@@ -0,0 +1,396 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesFactuality } from '../../src/matchers/llmGrading';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+import { createMockProvider } from '../factories/provider';
+import { mockProcessEnv } from '../util/utils';
+
+import type { GradingConfig } from '../../src/types/index';
+
+describe('matchesFactuality', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockReset();
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValue({
+      output:
+        '(A) The submitted answer is a subset of the expert answer and is fully consistent with it.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should pass when the factuality check passes with legacy format', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output:
+        '(A) The submitted answer is a subset of the expert answer and is fully consistent with it.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason:
+        'The submitted answer is a subset of the expert answer and is fully consistent with it.',
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should pass when the factuality check passes with JSON format', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output:
+        '{"category": "A", "reason": "The submitted answer is a subset of the expert answer and is fully consistent with it."}',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason:
+        'The submitted answer is a subset of the expert answer and is fully consistent with it.',
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should fall back to pattern match response', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '(A) This is a custom reason for category A.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason: 'This is a custom reason for category A.',
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should fail when the factuality check fails with legacy format', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '(D) There is a disagreement between the submitted answer and the expert answer.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: false,
+      reason: 'There is a disagreement between the submitted answer and the expert answer.',
+      score: 0,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should fail when the factuality check fails with JSON format', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output:
+        '{"category": "D", "reason": "There is a disagreement between the submitted answer and the expert answer."}',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: false,
+      reason: 'There is a disagreement between the submitted answer and the expert answer.',
+      score: 0,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should use the overridden factuality grading config', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {
+      factuality: {
+        subset: 0.8,
+        superset: 0.9,
+        agree: 1,
+        disagree: 0,
+        differButFactual: 0.7,
+      },
+    };
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output:
+        '{"category": "A", "reason": "The submitted answer is a subset of the expert answer and is fully consistent with it."}',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason:
+        'The submitted answer is a subset of the expert answer and is fully consistent with it.',
+      score: 0.8,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should use category description as fallback when no reason is provided in JSON', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '{"category": "A"}',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: true,
+      reason:
+        'Category A: The submitted answer is a subset of the expert answer and is fully consistent with it.',
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should fail when JSON has invalid category', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '{"category": "Z", "reason": "Invalid category"}',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(mockCallApi);
+
+    await expect(matchesFactuality(input, expected, output, grading)).resolves.toEqual({
+      pass: false,
+      score: 0,
+      reason: 'Invalid category value: Z',
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+  });
+
+  it('should use custom prompt override when provided', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+
+    const customPrompt = JSON.stringify([
+      {
+        role: 'system',
+        content: `You are comparing a submitted answer to an expert answer on a given question. Here is the data:
+[BEGIN DATA]
+************
+[Question]: {{input}}
+************
+[Expert]: {{ideal}}
+************
+[Submission]: {{completion}}
+************
+[END DATA]
+
+Compare the factual content of the submitted answer with the expert answer. Ignore any differences in style, grammar, or punctuation.
+The submitted answer may either be a subset or superset of the expert answer, or it may conflict with it. Determine which case applies. Answer the question by selecting one of the following options:
+(A) The submitted answer is a subset of the expert answer and is fully consistent with it.
+(B) The submitted answer is a superset of the expert answer and is fully consistent with it.
+(C) The submitted answer contains all the same details as the expert answer.
+(D) There is a disagreement between the submitted answer and the expert answer.
+(E) The answers differ, but these differences don't matter from the perspective of factuality.`,
+      },
+    ]);
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '(B) The submitted answer is a superset of the expert answer.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    const grading = {
+      rubricPrompt: customPrompt,
+      provider: createMockProvider({ callApi: mockCallApi }),
+    };
+
+    const result = await matchesFactuality(input, expected, output, grading);
+
+    expect(result).toEqual({
+      pass: true,
+      reason: 'The submitted answer is a superset of the expert answer.',
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+
+    // Verify the custom prompt was used
+    expect(mockCallApi).toHaveBeenCalledWith(
+      expect.stringContaining(
+        'The submitted answer may either be a subset or superset of the expert answer',
+      ),
+      expect.any(Object),
+    );
+  });
+
+  it('should throw an error when an error occurs', async () => {
+    const input = 'Input text';
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const grading = {};
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(() => {
+      throw new Error('An error occurred');
+    });
+
+    await expect(matchesFactuality(input, expected, output, grading)).rejects.toThrow(
+      'An error occurred',
+    );
+  });
+
+  it('should use Nunjucks templating when PROMPTFOO_DISABLE_TEMPLATING is set', async () => {
+    const restoreEnv = mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    try {
+      const input = 'Input {{ var }}';
+      const expected = 'Expected {{ var }}';
+      const output = 'Output {{ var }}';
+      const grading: GradingConfig = {
+        provider: DefaultGradingProvider,
+      };
+
+      vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValue({
+        output: '{"category": "A", "reason": "The submitted answer is correct."}',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+
+      await matchesFactuality(input, expected, output, grading);
+
+      expect(DefaultGradingProvider.callApi).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          vars: expect.objectContaining({
+            input: 'Input {{ var }}',
+            ideal: 'Expected {{ var }}',
+            completion: 'Output {{ var }}',
+          }),
+        }),
+      );
+    } finally {
+      restoreEnv();
+    }
+  });
+
+  it('should correctly substitute variables in custom rubricPrompt', async () => {
+    const input = 'What is the capital of France?';
+    const expected = 'Paris';
+    const output = 'The capital of France is Paris.';
+
+    const customPrompt = `Compare these answers:
+Question: {{input}}
+Reference: {{ideal}}
+Submitted: {{completion}}
+
+Determine if submitted answer is factually correct.
+Choose: (A) subset, (B) superset, (C) same, (D) disagree, (E) differ but factual`;
+
+    const mockCallApi = vi.fn().mockResolvedValue({
+      output: '(A) The submitted answer is correct.',
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+    });
+
+    const grading = {
+      rubricPrompt: customPrompt,
+      provider: createMockProvider({ callApi: mockCallApi }),
+    };
+
+    const result = await matchesFactuality(input, expected, output, grading);
+
+    expect(result).toEqual({
+      pass: true,
+      reason: 'The submitted answer is correct.',
+      score: 1,
+      tokensUsed: expect.objectContaining({
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+      }),
+    });
+
+    // Verify all variables were substituted in the prompt
+    expect(mockCallApi).toHaveBeenCalledTimes(1);
+    const actualPrompt = mockCallApi.mock.calls[0][0];
+    expect(actualPrompt).toContain('Question: What is the capital of France?');
+    expect(actualPrompt).toContain('Reference: Paris');
+    expect(actualPrompt).toContain('Submitted: The capital of France is Paris.');
+    expect(actualPrompt).not.toContain('{{input}}');
+    expect(actualPrompt).not.toContain('{{ideal}}');
+    expect(actualPrompt).not.toContain('{{completion}}');
+  });
+});

test/matchers/g-eval.test.ts

@@ -0,0 +1,607 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesGEval } from '../../src/matchers/llmGrading';
+import { resolveProvider } from '../../src/providers';
+import { createLiteLLMProvider } from '../../src/providers/litellm';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+
+describe('matchesGEval', () => {
+  let originalCallApi: typeof DefaultGradingProvider.callApi;
+
+  beforeEach(() => {
+    originalCallApi = DefaultGradingProvider.callApi;
+
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockImplementation(async (prompt) => {
+      if (prompt.includes('generate 3-4 concise evaluation steps')) {
+        return {
+          output: '{"steps": ["Check clarity", "Evaluate coherence", "Assess grammar"]}',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        };
+      } else {
+        return {
+          output: '{"score": 8, "reason": "The response is well-structured and clear"}',
+          tokenUsage: { total: 15, prompt: 8, completion: 7 },
+        };
+      }
+    });
+  });
+
+  afterEach(() => {
+    DefaultGradingProvider.callApi = originalCallApi;
+  });
+
+  it('should properly evaluate with default prompts', async () => {
+    const criteria = 'Evaluate coherence and clarity';
+    const input = 'Test input';
+    const output = 'Test output';
+    const threshold = 0.7;
+
+    const result = await matchesGEval(criteria, input, output, threshold);
+
+    expect(result).toEqual({
+      pass: true,
+      score: 0.8,
+      reason: 'The response is well-structured and clear',
+      tokensUsed: expect.any(Object),
+    });
+
+    expect(DefaultGradingProvider.callApi).toHaveBeenCalledTimes(2);
+  });
+
+  it.each([
+    ['positive', 'the answer should contain harmful or inappropriate content'],
+    ['negated', 'the answer should not contain harmful or inappropriate content'],
+  ])('should render observed-condition scoring for %s criteria', async (_, criteria) => {
+    await matchesGEval(criteria, 'Test input', 'Test output', 0.7);
+
+    expect(DefaultGradingProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(DefaultGradingProvider.callApi).toHaveBeenNthCalledWith(
+      1,
+      expect.stringContaining(criteria),
+      expect.any(Object),
+    );
+    expect(DefaultGradingProvider.callApi).toHaveBeenNthCalledWith(
+      2,
+      expect.stringContaining(criteria),
+      expect.any(Object),
+    );
+    expect(DefaultGradingProvider.callApi).toHaveBeenNthCalledWith(
+      2,
+      expect.stringContaining(
+        'the condition described by the Evaluation Criteria is fully and clearly observed in the Reply',
+      ),
+      expect.any(Object),
+    );
+    expect(DefaultGradingProvider.callApi).toHaveBeenNthCalledWith(
+      2,
+      expect.stringContaining('and 0 indicates that it is not observed at all'),
+      expect.any(Object),
+    );
+  });
+
+  it('should handle custom rubric prompts', async () => {
+    vi.resetAllMocks();
+
+    const mockCallApi = vi
+      .fn()
+      .mockImplementationOnce(() => ({
+        output: '{"steps": ["Custom step 1", "Custom step 2"]}',
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      }))
+      .mockImplementationOnce(() => ({
+        output: '{"score": 8, "reason": "Custom evaluation complete", "pass": true}',
+        tokenUsage: { total: 15, prompt: 8, completion: 7 },
+      }));
+
+    DefaultGradingProvider.callApi = mockCallApi;
+
+    const criteria = 'Evaluate coherence and clarity';
+    const input = 'Test input';
+    const output = 'Test output';
+    const threshold = 0.7;
+
+    const grading = {
+      rubricPrompt: {
+        steps: 'Custom steps template with {{criteria}}',
+        evaluate: 'Custom evaluation template with {{criteria}} and {{steps}}',
+      },
+    } as any;
+
+    const result = await matchesGEval(criteria, input, output, threshold, grading);
+
+    expect(result.score).toBe(0.8);
+
+    expect(mockCallApi).toHaveBeenCalledTimes(2);
+    expect(mockCallApi).toHaveBeenNthCalledWith(
+      1,
+      expect.stringContaining('Custom steps template with'),
+      expect.any(Object),
+    );
+    expect(mockCallApi).toHaveBeenNthCalledWith(
+      2,
+      expect.stringContaining('Custom evaluation template with'),
+      expect.any(Object),
+    );
+
+    DefaultGradingProvider.callApi = originalCallApi;
+  });
+
+  it('passes LiteLLM provider context through both G-Eval phases', async () => {
+    const criteria = 'Reward factual grounding and completeness';
+    const configuredLiteLLM = createLiteLLMProvider('litellm:gemini-pro', {
+      config: {
+        config: {
+          apiBaseUrl: 'http://litellm-regression.local:4000',
+          apiKey: 'test-litellm-key',
+          temperature: 0,
+        },
+      },
+    });
+
+    const resolvedProvider = await resolveProvider('litellm:gemini-pro', {
+      [configuredLiteLLM.id()]: configuredLiteLLM,
+    });
+    expect(resolvedProvider).toBe(configuredLiteLLM);
+    expect(configuredLiteLLM.config.apiBaseUrl).toBe('http://litellm-regression.local:4000');
+
+    const litellmCallApi = vi
+      .fn()
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check factual alignment", "Check completeness"]}',
+        tokenUsage: { total: 11, prompt: 6, completion: 5 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output: '{"score": 8, "reason": "The answer is grounded and complete"}',
+        tokenUsage: { total: 13, prompt: 7, completion: 6 },
+      }));
+    resolvedProvider.callApi = litellmCallApi;
+
+    const result = await matchesGEval(
+      criteria,
+      'Question: what is the capital of France?',
+      'Paris is the capital of France.',
+      0.7,
+      { provider: resolvedProvider },
+    );
+
+    expect(result).toEqual({
+      pass: true,
+      score: 0.8,
+      reason: 'The answer is grounded and complete',
+      tokensUsed: expect.objectContaining({ total: 24, prompt: 13, completion: 11 }),
+    });
+    expect(DefaultGradingProvider.callApi).not.toHaveBeenCalled();
+    expect(litellmCallApi).toHaveBeenCalledTimes(2);
+    expect(litellmCallApi).toHaveBeenNthCalledWith(
+      1,
+      expect.stringContaining('generate 3-4 concise evaluation steps'),
+      expect.objectContaining({
+        prompt: expect.objectContaining({ label: 'g-eval-steps' }),
+        vars: { criteria },
+      }),
+    );
+    expect(litellmCallApi).toHaveBeenNthCalledWith(
+      2,
+      expect.stringContaining('Check factual alignment'),
+      expect.objectContaining({
+        prompt: expect.objectContaining({ label: 'g-eval' }),
+        vars: expect.objectContaining({
+          criteria,
+          input: 'Question: what is the capital of France?',
+          maxScore: '10',
+          output: 'Paris is the capital of France.',
+        }),
+      }),
+    );
+    expect(litellmCallApi.mock.calls[1][1].vars.steps).toContain('Check completeness');
+  });
+
+  it('should fail when score is below threshold', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => {
+        return {
+          output: '{"steps": ["Check clarity", "Evaluate coherence", "Assess grammar"]}',
+          tokenUsage: { total: 10, prompt: 5, completion: 5 },
+        };
+      })
+      .mockImplementationOnce(async () => {
+        return {
+          output: '{"score": 3, "reason": "The response lacks coherence"}',
+          tokenUsage: { total: 15, prompt: 8, completion: 7 },
+        };
+      });
+
+    const criteria = 'Evaluate coherence and clarity';
+    const input = 'Test input';
+    const output = 'Test output';
+    const threshold = 0.7;
+
+    const result = await matchesGEval(criteria, input, output, threshold);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0.3,
+      reason: 'The response lacks coherence',
+      tokensUsed: expect.any(Object),
+    });
+  });
+
+  it('should return provider errors from the step-generation call', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      error: 'steps provider unavailable',
+      tokenUsage: { total: 3, prompt: 1, completion: 2 },
+    });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'steps provider unavailable',
+      tokensUsed: expect.objectContaining({
+        total: 3,
+        prompt: 1,
+        completion: 2,
+      }),
+      metadata: { graderError: true },
+    });
+    expect(DefaultGradingProvider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('reports the LiteLLM grader and phase when step generation returns no output', async () => {
+    const provider = createLiteLLMProvider('litellm:gemini-pro', {});
+    provider.callApi = vi.fn().mockResolvedValueOnce({
+      output: '',
+      tokenUsage: { total: 3, prompt: 1, completion: 2 },
+    });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7, {
+      provider,
+    });
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval step-generation call to litellm:gemini-pro returned no output',
+      tokensUsed: expect.objectContaining({ total: 3, prompt: 1, completion: 2 }),
+      metadata: { graderError: true },
+    });
+    expect(DefaultGradingProvider.callApi).not.toHaveBeenCalled();
+    expect(provider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('should fail clearly when the evaluation steps shape is invalid', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      output: '{"steps":"Check clarity"}',
+      tokenUsage: { total: 3, prompt: 1, completion: 2 },
+    });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval steps response has invalid or missing steps: "Check clarity"',
+      tokensUsed: expect.objectContaining({
+        total: 3,
+        prompt: 1,
+        completion: 2,
+      }),
+      metadata: { graderError: true },
+    });
+    expect(DefaultGradingProvider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('should return provider errors from the evaluation call', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        error: 'evaluation provider unavailable',
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'evaluation provider unavailable',
+      tokensUsed: expect.objectContaining({
+        total: 7,
+        prompt: 3,
+        completion: 4,
+      }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it('reports the LiteLLM grader and phase when evaluation returns no output', async () => {
+    const provider = createLiteLLMProvider('litellm:gemini-pro', {});
+    provider.callApi = vi
+      .fn()
+      .mockResolvedValueOnce({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      })
+      .mockResolvedValueOnce({
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7, {
+      provider,
+    });
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval evaluation call to litellm:gemini-pro returned no output',
+      tokensUsed: expect.objectContaining({ total: 7, prompt: 3, completion: 4 }),
+      metadata: { graderError: true },
+    });
+    expect(DefaultGradingProvider.callApi).not.toHaveBeenCalled();
+    expect(provider.callApi).toHaveBeenCalledTimes(2);
+  });
+
+  it('should fail clearly when the evaluation result is not a string', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output: { score: 8, reason: 'object output' },
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'LLM-proposed evaluation result response is not a string',
+      tokensUsed: expect.objectContaining({
+        total: 7,
+      }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it.each([
+    ['null score', '{"score": null, "reason": "null score"}', 'null'],
+    ['blank string score', '{"score": "", "reason": "blank score"}', '""'],
+  ])('should fail clearly when the evaluation score is %s', async (_, output, scoreLabel) => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output,
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: `G-Eval result has invalid or missing score: ${scoreLabel}`,
+      tokensUsed: expect.objectContaining({
+        total: 7,
+      }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it('should fail clearly when the evaluation score is outside the expected range', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output: '{"score": 11, "reason": "too high"}',
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval result score 11 is outside the expected 0-10 range',
+      tokensUsed: expect.objectContaining({
+        total: 7,
+      }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it('should fail clearly when the evaluation reason is missing', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output: '{"score": 8}',
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval result has invalid or missing reason: undefined',
+      tokensUsed: expect.objectContaining({
+        total: 7,
+      }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it.each([
+    ['null reason', '{"score": 8, "reason": null}', 'null'],
+    ['numeric reason', '{"score": 8, "reason": 42}', '42'],
+    ['empty string reason', '{"score": 8, "reason": ""}', '""'],
+    ['whitespace-only reason', '{"score": 8, "reason": "   "}', '"   "'],
+  ])('should fail clearly when the evaluation reason is %s', async (_, output, reasonLabel) => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output,
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: `G-Eval result has invalid or missing reason: ${reasonLabel}`,
+      tokensUsed: expect.objectContaining({ total: 7, prompt: 3, completion: 4 }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it('should fail clearly when the steps response contains non-string elements', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      output: '{"steps": ["ok", 42]}',
+      tokenUsage: { total: 3, prompt: 1, completion: 2 },
+    });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval steps response contains invalid steps: ["ok",42]',
+      tokensUsed: expect.objectContaining({ total: 3, prompt: 1, completion: 2 }),
+      metadata: { graderError: true },
+    });
+    expect(DefaultGradingProvider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('should fail clearly when a steps entry is whitespace-only', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      output: '{"steps": ["ok", "   "]}',
+      tokenUsage: { total: 3, prompt: 1, completion: 2 },
+    });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval steps response contains invalid steps: ["ok","   "]',
+      tokensUsed: expect.objectContaining({ total: 3, prompt: 1, completion: 2 }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it('should fail clearly when the steps array is empty', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi').mockResolvedValueOnce({
+      output: '{"steps": []}',
+      tokenUsage: { total: 3, prompt: 1, completion: 2 },
+    });
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'LLM does not propose any evaluation step',
+      tokensUsed: expect.objectContaining({ total: 3, prompt: 1, completion: 2 }),
+      metadata: { graderError: true },
+    });
+    expect(DefaultGradingProvider.callApi).toHaveBeenCalledTimes(1);
+  });
+
+  it('should accept numeric-string scores', async () => {
+    // Positive path for the string coercion branch in rawScore parsing —
+    // locks in that a valid numeric string still produces a real grade so a
+    // future tightening doesn't silently reject all string scores.
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output: '{"score": "  7  ", "reason": "fine"}',
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result.pass).toBe(true);
+    expect(result.score).toBeCloseTo(0.7, 5);
+    expect(result.reason).toBe('fine');
+    expect(result.metadata).toBeUndefined();
+  });
+
+  it.each([
+    ['lower bound exactly (0)', '{"score": 0, "reason": "absent"}', 0],
+    ['upper bound exactly (10)', '{"score": 10, "reason": "perfect"}', 1],
+  ])('should accept score at the %s', async (_, output, expectedScore) => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output,
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result.score).toBeCloseTo(expectedScore, 5);
+    expect(result.metadata).toBeUndefined();
+  });
+
+  it('should fail clearly when the evaluation score is negative', async () => {
+    vi.spyOn(DefaultGradingProvider, 'callApi')
+      .mockImplementationOnce(async () => ({
+        output: '{"steps": ["Check clarity"]}',
+        tokenUsage: { total: 3, prompt: 1, completion: 2 },
+      }))
+      .mockImplementationOnce(async () => ({
+        output: '{"score": -1, "reason": "too low"}',
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }));
+
+    const result = await matchesGEval('Evaluate coherence', 'Test input', 'Test output', 0.7);
+
+    expect(result).toEqual({
+      pass: false,
+      score: 0,
+      reason: 'G-Eval result score -1 is outside the expected 0-10 range',
+      tokensUsed: expect.objectContaining({ total: 7, prompt: 3, completion: 4 }),
+      metadata: { graderError: true },
+    });
+  });
+
+  it('tracks token usage for both API calls', async () => {
+    const criteria = 'Evaluate coherence and clarity';
+    const input = 'Test input';
+    const output = 'Test output';
+    const threshold = 0.7;
+
+    const result = await matchesGEval(criteria, input, output, threshold);
+
+    expect(result.tokensUsed).toEqual({
+      total: 25,
+      prompt: 13,
+      completion: 12,
+      cached: 0,
+      completionDetails: expect.any(Object),
+      numRequests: 0,
+    });
+  });
+});

test/matchers/getGradingProvider.test.ts

@@ -0,0 +1,401 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import cliState from '../../src/cliState';
+import { getGradingProvider } from '../../src/matchers/providers';
+import { loadApiProvider } from '../../src/providers/index';
+import { createMockProvider } from '../factories/provider';
+
+vi.mock('../../src/providers', () => ({
+  loadApiProvider: vi.fn(),
+}));
+
+vi.mock('../../src/cliState');
+
+describe('getGradingProvider', () => {
+  const mockProvider = createMockProvider();
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+    (cliState as any).config = {};
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  describe('explicit provider parameter', () => {
+    it('should use provider when specified as string', async () => {
+      vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
+
+      const result = await getGradingProvider('text', 'openai:gpt-4', null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('openai:gpt-4', { basePath: undefined });
+      expect(result).toBe(mockProvider);
+    });
+
+    it('should use provider when specified as ApiProvider object', async () => {
+      const result = await getGradingProvider('text', mockProvider, null);
+
+      expect(result).toBe(mockProvider);
+      expect(loadApiProvider).not.toHaveBeenCalled();
+    });
+
+    it('should treat null provider as unspecified', async () => {
+      const result = await getGradingProvider('text', null as any, mockProvider);
+
+      expect(result).toBe(mockProvider);
+      expect(loadApiProvider).not.toHaveBeenCalled();
+    });
+
+    it('should use provider when specified as ProviderOptions', async () => {
+      const providerOptions = {
+        id: 'openai:gpt-4',
+        config: {
+          temperature: 0.5,
+        },
+      };
+      vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
+
+      const result = await getGradingProvider('text', providerOptions, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('openai:gpt-4', {
+        options: providerOptions,
+        basePath: undefined,
+      });
+      expect(result).toBe(mockProvider);
+    });
+  });
+
+  describe('ProviderTypeMap (embedding/classification/text record)', () => {
+    it('should handle embedding provider from ProviderTypeMap', async () => {
+      const providerMap = {
+        embedding: 'openai:embedding',
+      };
+      vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
+
+      const result = await getGradingProvider('embedding', providerMap, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('openai:embedding', { basePath: undefined });
+      expect(result).toBe(mockProvider);
+    });
+
+    it('should handle classification provider from ProviderTypeMap', async () => {
+      const providerMap = {
+        classification: 'openai:gpt-4',
+      };
+      vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
+
+      const result = await getGradingProvider('classification', providerMap, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('openai:gpt-4', { basePath: undefined });
+      expect(result).toBe(mockProvider);
+    });
+
+    it('should handle text provider from ProviderTypeMap', async () => {
+      const providerMap = {
+        text: 'anthropic:claude-3-sonnet',
+      };
+      vi.mocked(loadApiProvider).mockResolvedValue(mockProvider);
+
+      const result = await getGradingProvider('text', providerMap, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('anthropic:claude-3-sonnet', {
+        basePath: undefined,
+      });
+      expect(result).toBe(mockProvider);
+    });
+  });
+
+  describe('defaultTest.options.provider fallback', () => {
+    it('should use defaultTest.options.provider when no provider specified', async () => {
+      const azureProvider = createMockProvider({ id: 'azureopenai:chat:gpt-4' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          options: {
+            provider: 'azureopenai:chat:gpt-4',
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(azureProvider);
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('azureopenai:chat:gpt-4', {
+        basePath: undefined,
+      });
+      expect(result).toBe(azureProvider);
+    });
+
+    it('should use defaultTest.provider when options.provider not specified', async () => {
+      const azureProvider = createMockProvider({ id: 'azureopenai:chat:gpt-4' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          provider: 'azureopenai:chat:gpt-4',
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(azureProvider);
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('azureopenai:chat:gpt-4', {
+        basePath: undefined,
+      });
+      expect(result).toBe(azureProvider);
+    });
+
+    it('should skip defaultTest.provider when it is promptfoo:simulated-user', async () => {
+      const defaultProvider = createMockProvider({ id: 'default-provider' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          provider: {
+            id: 'promptfoo:simulated-user',
+            config: {
+              maxTurns: 3,
+            },
+          },
+        },
+      };
+
+      const result = await getGradingProvider('text', undefined, defaultProvider);
+
+      expect(loadApiProvider).not.toHaveBeenCalled();
+      expect(result).toBe(defaultProvider);
+    });
+
+    it('should fall back to defaultTest.options.provider when defaultTest.provider is promptfoo:simulated-user', async () => {
+      const azureProvider = createMockProvider({ id: 'azureopenai:chat:gpt-4' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          provider: {
+            id: 'promptfoo:simulated-user',
+            config: {
+              maxTurns: 3,
+            },
+          },
+          options: {
+            provider: 'azureopenai:chat:gpt-4',
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(azureProvider);
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('azureopenai:chat:gpt-4', {
+        basePath: undefined,
+      });
+      expect(result).toBe(azureProvider);
+    });
+
+    it('should use defaultTest.options.provider.text when specified', async () => {
+      const azureProvider = createMockProvider({ id: 'azureopenai:chat:gpt-4' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          options: {
+            provider: {
+              text: 'azureopenai:chat:gpt-4',
+            },
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(azureProvider);
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('azureopenai:chat:gpt-4', {
+        basePath: undefined,
+      });
+      expect(result).toBe(azureProvider);
+    });
+
+    it('should prefer defaultTest.provider over defaultTest.options.provider', async () => {
+      const azureProvider = createMockProvider({ id: 'azureopenai:chat:gpt-4' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          provider: 'azureopenai:chat:gpt-4',
+          options: {
+            provider: 'openai:gpt-4',
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(azureProvider);
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('azureopenai:chat:gpt-4', {
+        basePath: undefined,
+      });
+      expect(result).toBe(azureProvider);
+    });
+
+    it('should fall back to defaultProvider when no defaultTest provider configured', async () => {
+      const defaultProvider = createMockProvider({ id: 'default-provider' });
+
+      (cliState as any).config = {
+        defaultTest: {},
+      };
+
+      const result = await getGradingProvider('text', undefined, defaultProvider);
+
+      expect(loadApiProvider).not.toHaveBeenCalled();
+      expect(result).toBe(defaultProvider);
+    });
+
+    it('should fall back to defaultProvider when cliState.config is undefined', async () => {
+      const defaultProvider = createMockProvider({ id: 'default-provider' });
+
+      (cliState as any).config = undefined;
+
+      const result = await getGradingProvider('text', undefined, defaultProvider);
+
+      expect(loadApiProvider).not.toHaveBeenCalled();
+      expect(result).toBe(defaultProvider);
+    });
+
+    it('should return null when no provider and no defaultProvider specified', async () => {
+      (cliState as any).config = {};
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      expect(result).toBeNull();
+    });
+
+    it('should work with full Azure provider configuration', async () => {
+      const azureProvider = createMockProvider({ id: 'azureopenai:chat:gpt-4o' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          options: {
+            provider: {
+              id: 'azureopenai:chat:gpt-4o',
+              config: {
+                apiHost: 'https://my-resource.openai.azure.com',
+                apiKey: 'my-api-key',
+                deploymentName: 'gpt-4o',
+              },
+            },
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(azureProvider);
+
+      const result = await getGradingProvider('text', undefined, null);
+
+      // Since we recursively call getGradingProvider, it delegates to loadFromProviderOptions
+      // which calls loadApiProvider with the id and options structure
+      expect(loadApiProvider).toHaveBeenCalledWith('azureopenai:chat:gpt-4o', {
+        options: {
+          id: 'azureopenai:chat:gpt-4o',
+          config: {
+            apiHost: 'https://my-resource.openai.azure.com',
+            apiKey: 'my-api-key',
+            deploymentName: 'gpt-4o',
+          },
+        },
+        basePath: undefined,
+      });
+      expect(result).toBe(azureProvider);
+    });
+  });
+
+  describe('explicit provider takes precedence over defaultTest', () => {
+    it('should use explicit provider over defaultTest.options.provider', async () => {
+      const explicitProvider = createMockProvider({ id: 'explicit-provider' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          options: {
+            provider: 'azureopenai:chat:gpt-4',
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(explicitProvider);
+
+      const result = await getGradingProvider('text', 'openai:gpt-4o', null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('openai:gpt-4o', { basePath: undefined });
+      expect(result).toBe(explicitProvider);
+    });
+
+    it('should use explicit provider object over defaultTest', async () => {
+      const explicitProvider = createMockProvider({ id: 'explicit-provider' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          options: {
+            provider: 'azureopenai:chat:gpt-4',
+          },
+        },
+      };
+
+      const result = await getGradingProvider('text', explicitProvider, null);
+
+      expect(loadApiProvider).not.toHaveBeenCalled();
+      expect(result).toBe(explicitProvider);
+    });
+  });
+
+  describe('error handling', () => {
+    it('should throw error when provider is an array', async () => {
+      const providerArray = ['openai:gpt-4'];
+
+      await expect(getGradingProvider('text', providerArray as any, null)).rejects.toThrow(
+        'Provider must be an object or string, but received an array',
+      );
+    });
+
+    it('should throw error for invalid provider definition', async () => {
+      const invalidProvider = { foo: 'bar' };
+
+      await expect(getGradingProvider('text', invalidProvider as any, null)).rejects.toThrow(
+        "Invalid provider definition for output type 'text'",
+      );
+    });
+  });
+
+  describe('backwards compatibility', () => {
+    it('should maintain existing behavior when defaultTest not configured', async () => {
+      const defaultProvider = createMockProvider({ id: 'default-provider' });
+
+      // No defaultTest in config
+      (cliState as any).config = {};
+
+      const result = await getGradingProvider('text', undefined, defaultProvider);
+
+      expect(result).toBe(defaultProvider);
+      expect(loadApiProvider).not.toHaveBeenCalled();
+    });
+
+    it('should maintain existing behavior with explicit provider', async () => {
+      const explicitProvider = createMockProvider({ id: 'explicit-provider' });
+
+      (cliState as any).config = {
+        defaultTest: {
+          options: {
+            provider: 'should-be-ignored',
+          },
+        },
+      };
+
+      vi.mocked(loadApiProvider).mockResolvedValue(explicitProvider);
+
+      const result = await getGradingProvider('text', 'openai:gpt-4', null);
+
+      expect(loadApiProvider).toHaveBeenCalledWith('openai:gpt-4', { basePath: undefined });
+      expect(result).toBe(explicitProvider);
+    });
+  });
+});

test/matchers/max-score.test.ts

@@ -0,0 +1,436 @@
+import { describe, expect, it } from 'vitest';
+import { selectMaxScore } from '../../src/matchers/comparison';
+import { createEvaluateResult } from '../factories/eval';
+import { createGradingResult } from '../factories/gradingResult';
+import { createPrompt } from '../factories/testSuite';
+
+import type { Assertion, EvaluateResult } from '../../src/types/index';
+
+describe('selectMaxScore', () => {
+  const createMockResult = (score: number, testIdx: number): EvaluateResult =>
+    createEvaluateResult({
+      testIdx,
+      testCase: {
+        assert: [
+          { type: 'contains', value: 'apple' },
+          { type: 'contains', value: 'orange' },
+          { type: 'max-score' },
+        ],
+      },
+      prompt: createPrompt('test prompt', { label: 'test' }),
+      promptId: 'prompt-test',
+      response: { output: `Output ${testIdx}` },
+      score,
+      latencyMs: 100,
+      gradingResult: createGradingResult({
+        pass: score > 0.5,
+        score,
+        reason: 'Test result',
+        namedScores: {},
+        tokensUsed: { total: 0, prompt: 0, completion: 0, cached: 0 },
+        componentResults: [
+          {
+            pass: true,
+            score: 1,
+            reason: 'Contains apple',
+            assertion: { type: 'contains', value: 'apple' } as Assertion,
+          },
+          {
+            pass: score === 1,
+            score: score === 1 ? 1 : 0,
+            reason: score === 1 ? 'Contains orange' : 'Does not contain orange',
+            assertion: { type: 'contains', value: 'orange' } as Assertion,
+          },
+        ],
+      }),
+      metadata: {},
+    });
+
+  const mockAssertion: Assertion = {
+    type: 'max-score',
+  };
+
+  it('should select the output with the highest score', async () => {
+    const outputs = ['Output 0', 'Output 1', 'Output 2'];
+    const results = [createMockResult(0.5, 0), createMockResult(1.0, 1), createMockResult(0.75, 2)];
+
+    const gradingResults = await selectMaxScore(outputs, results, mockAssertion);
+
+    expect(gradingResults).toHaveLength(3);
+    expect(gradingResults[0].pass).toBe(false);
+    expect(gradingResults[0].score).toBe(0);
+    expect(gradingResults[0].reason).toContain('Not selected');
+
+    expect(gradingResults[1].pass).toBe(true);
+    expect(gradingResults[1].score).toBe(1);
+    expect(gradingResults[1].reason).toContain('Selected as highest scoring output');
+
+    expect(gradingResults[2].pass).toBe(false);
+    expect(gradingResults[2].score).toBe(0);
+    expect(gradingResults[2].reason).toContain('Not selected');
+  });
+
+  it('should handle ties by selecting the first output', async () => {
+    const outputs = ['Output 0', 'Output 1', 'Output 2'];
+    const results = [createMockResult(1.0, 0), createMockResult(1.0, 1), createMockResult(0.5, 2)];
+
+    const gradingResults = await selectMaxScore(outputs, results, mockAssertion);
+
+    expect(gradingResults[0].pass).toBe(true);
+    expect(gradingResults[0].reason).toContain('Selected as highest scoring output');
+
+    expect(gradingResults[1].pass).toBe(false);
+    expect(gradingResults[1].reason).toContain('Not selected (score: 1.000, max: 1.000)');
+
+    expect(gradingResults[2].pass).toBe(false);
+  });
+
+  it('should apply threshold when specified', async () => {
+    const outputs = ['Output 0', 'Output 1'];
+    const results = [createMockResult(0.6, 0), createMockResult(0.4, 1)];
+
+    const assertionWithThreshold: Assertion = {
+      type: 'max-score',
+      value: {
+        threshold: 0.7,
+      },
+    };
+
+    const gradingResults = await selectMaxScore(outputs, results, assertionWithThreshold);
+
+    expect(gradingResults[0].pass).toBe(false);
+    expect(gradingResults[0].reason).toContain('below threshold');
+
+    expect(gradingResults[1].pass).toBe(false);
+  });
+
+  it('should throw error with fewer than 2 outputs', async () => {
+    const outputs = ['Output 0'];
+    const results = [createMockResult(1.0, 0)];
+
+    await expect(selectMaxScore(outputs, results, mockAssertion)).rejects.toThrow(
+      'max-score assertion must have at least two outputs to compare between',
+    );
+  });
+
+  it('should throw error when no other assertions exist', async () => {
+    const outputs = ['Output 0', 'Output 1'];
+    const results = [
+      {
+        ...createMockResult(1.0, 0),
+        testCase: {
+          assert: [{ type: 'max-score' }],
+        },
+        gradingResult: {
+          ...createMockResult(1.0, 0).gradingResult,
+          componentResults: [], // Empty component results
+        },
+      },
+      {
+        ...createMockResult(0.5, 1),
+        testCase: {
+          assert: [{ type: 'max-score' }],
+        },
+        gradingResult: {
+          ...createMockResult(0.5, 1).gradingResult,
+          componentResults: [], // Empty component results
+        },
+      },
+    ];
+
+    await expect(selectMaxScore(outputs, results, mockAssertion)).rejects.toThrow(
+      'max-score requires at least one other assertion (besides max-score or select-best) to aggregate scores from',
+    );
+  });
+
+  it('should exclude select-best assertions from validation', async () => {
+    const outputs = ['Output 0', 'Output 1'];
+    const results = [
+      {
+        ...createMockResult(1.0, 0),
+        testCase: {
+          assert: [
+            { type: 'contains', value: 'test' },
+            { type: 'select-best' },
+            { type: 'max-score' },
+          ],
+        },
+      },
+      {
+        ...createMockResult(0.5, 1),
+        testCase: {
+          assert: [
+            { type: 'contains', value: 'test' },
+            { type: 'select-best' },
+            { type: 'max-score' },
+          ],
+        },
+      },
+    ];
+
+    const gradingResults = await selectMaxScore(outputs, results, mockAssertion);
+
+    expect(gradingResults).toHaveLength(2);
+    expect(gradingResults[0].pass).toBe(true);
+    expect(gradingResults[1].pass).toBe(false);
+  });
+
+  it('should handle results with zero scores', async () => {
+    const outputs = ['Output 0', 'Output 1', 'Output 2'];
+    // Create results with different component scores
+    const results = [
+      {
+        ...createMockResult(0, 0),
+        gradingResult: {
+          ...createMockResult(0, 0).gradingResult,
+          componentResults: [
+            {
+              pass: false,
+              score: 0,
+              reason: 'Does not contain apple',
+              assertion: { type: 'contains', value: 'apple' } as Assertion,
+            },
+            {
+              pass: false,
+              score: 0,
+              reason: 'Does not contain orange',
+              assertion: { type: 'contains', value: 'orange' } as Assertion,
+            },
+          ],
+        },
+      },
+      {
+        ...createMockResult(0, 1),
+        gradingResult: {
+          ...createMockResult(0, 1).gradingResult,
+          componentResults: [
+            {
+              pass: false,
+              score: 0,
+              reason: 'Does not contain apple',
+              assertion: { type: 'contains', value: 'apple' } as Assertion,
+            },
+            {
+              pass: false,
+              score: 0,
+              reason: 'Does not contain orange',
+              assertion: { type: 'contains', value: 'orange' } as Assertion,
+            },
+          ],
+        },
+      },
+      {
+        ...createMockResult(0.1, 2),
+        gradingResult: {
+          ...createMockResult(0.1, 2).gradingResult,
+          componentResults: [
+            {
+              pass: true,
+              score: 0.1,
+              reason: 'Partially contains apple',
+              assertion: { type: 'contains', value: 'apple' } as Assertion,
+            },
+            {
+              pass: false,
+              score: 0,
+              reason: 'Does not contain orange',
+              assertion: { type: 'contains', value: 'orange' } as Assertion,
+            },
+          ],
+        },
+      },
+    ];
+
+    const gradingResults = await selectMaxScore(outputs, results, mockAssertion);
+
+    expect(gradingResults[0].pass).toBe(false);
+    expect(gradingResults[1].pass).toBe(false);
+    expect(gradingResults[2].pass).toBe(true);
+    expect(gradingResults[2].reason).toContain('Selected as highest scoring output (score: 0.050)');
+  });
+
+  it('should include score in namedScores', async () => {
+    const outputs = ['Output 0', 'Output 1'];
+    const results = [createMockResult(0.8, 0), createMockResult(0.6, 1)];
+
+    const gradingResults = await selectMaxScore(outputs, results, mockAssertion);
+
+    // Both results have the same componentResults scores (1 for apple, 0 for orange)
+    // So both have average score of 0.5
+    expect(gradingResults[0].namedScores?.maxScore).toBe(0.5);
+    expect(gradingResults[1].namedScores?.maxScore).toBe(0.5);
+    // The first one should be selected due to tie-breaking
+    expect(gradingResults[0].pass).toBe(true);
+    expect(gradingResults[1].pass).toBe(false);
+  });
+
+  describe('Threshold Edge Cases - Truthy vs Undefined', () => {
+    it('should handle threshold=0 differently from threshold=undefined', async () => {
+      const outputs = ['Output 0', 'Output 1'];
+      const results = [
+        createMockResult(0, 0), // Max score will be 0 (average of 1 and 0)
+        createMockResult(0.5, 1), // Lower score
+      ];
+
+      // Test with threshold=undefined (no threshold)
+      const assertionWithoutThreshold: Assertion = {
+        type: 'max-score',
+        value: {},
+      };
+
+      const resultsWithoutThreshold = await selectMaxScore(
+        outputs,
+        results,
+        assertionWithoutThreshold,
+      );
+
+      // With no threshold, should select the winner (index 0 with score 0.5)
+      expect(resultsWithoutThreshold[0].pass).toBe(true);
+      expect(resultsWithoutThreshold[1].pass).toBe(false);
+
+      // Test with threshold=0
+      const assertionWithZeroThreshold: Assertion = {
+        type: 'max-score',
+        value: {
+          threshold: 0,
+        },
+      };
+
+      const resultsWithZeroThreshold = await selectMaxScore(
+        outputs,
+        results,
+        assertionWithZeroThreshold,
+      );
+
+      // With threshold=0, max score is 0.5 which is >= 0, so should still pass
+      expect(resultsWithZeroThreshold[0].pass).toBe(true);
+      expect(resultsWithZeroThreshold[1].pass).toBe(false);
+    });
+
+    it('should fail all outputs when max score is below threshold=0', async () => {
+      const outputs = ['Output 0', 'Output 1'];
+
+      // Create results where all scores are negative (so max will be negative)
+      const results = [
+        {
+          ...createMockResult(-0.5, 0),
+          gradingResult: {
+            ...createMockResult(-0.5, 0).gradingResult,
+            componentResults: [
+              {
+                pass: false,
+                score: -1, // Negative score
+                reason: 'Negative test result',
+                assertion: { type: 'contains', value: 'apple' } as Assertion,
+              },
+              {
+                pass: false,
+                score: 0,
+                reason: 'Zero test result',
+                assertion: { type: 'contains', value: 'orange' } as Assertion,
+              },
+            ],
+          },
+        },
+        {
+          ...createMockResult(-1, 1),
+          gradingResult: {
+            ...createMockResult(-1, 1).gradingResult,
+            componentResults: [
+              {
+                pass: false,
+                score: -2, // Even more negative
+                reason: 'Very negative test result',
+                assertion: { type: 'contains', value: 'apple' } as Assertion,
+              },
+              {
+                pass: false,
+                score: 0,
+                reason: 'Zero test result',
+                assertion: { type: 'contains', value: 'orange' } as Assertion,
+              },
+            ],
+          },
+        },
+      ];
+
+      const assertionWithZeroThreshold: Assertion = {
+        type: 'max-score',
+        value: {
+          threshold: 0,
+        },
+      };
+
+      const gradingResults = await selectMaxScore(outputs, results, assertionWithZeroThreshold);
+
+      // Max score will be -0.5, which is < 0, so all should fail
+      expect(gradingResults[0].pass).toBe(false);
+      expect(gradingResults[1].pass).toBe(false);
+      // The winner (index 0) should get the threshold message since it has max score but doesn't meet threshold
+      expect(gradingResults[0].reason).toContain('below threshold');
+      // The loser (index 1) gets the regular "not selected" message
+      expect(gradingResults[1].reason).toContain('Not selected');
+    });
+
+    it('should pass winner when max score equals threshold=0', async () => {
+      const outputs = ['Output 0', 'Output 1'];
+
+      // Create results where max score will be exactly 0
+      const results = [
+        {
+          ...createMockResult(0, 0),
+          gradingResult: {
+            ...createMockResult(0, 0).gradingResult,
+            componentResults: [
+              {
+                pass: false,
+                score: 0,
+                reason: 'Zero score',
+                assertion: { type: 'contains', value: 'apple' } as Assertion,
+              },
+              {
+                pass: false,
+                score: 0,
+                reason: 'Zero score',
+                assertion: { type: 'contains', value: 'orange' } as Assertion,
+              },
+            ],
+          },
+        },
+        {
+          ...createMockResult(-1, 1),
+          gradingResult: {
+            ...createMockResult(-1, 1).gradingResult,
+            componentResults: [
+              {
+                pass: false,
+                score: -2,
+                reason: 'Negative score',
+                assertion: { type: 'contains', value: 'apple' } as Assertion,
+              },
+              {
+                pass: false,
+                score: 2, // This balances to average of 0
+                reason: 'Positive score',
+                assertion: { type: 'contains', value: 'orange' } as Assertion,
+              },
+            ],
+          },
+        },
+      ];
+
+      const assertionWithZeroThreshold: Assertion = {
+        type: 'max-score',
+        value: {
+          threshold: 0,
+        },
+      };
+
+      const gradingResults = await selectMaxScore(outputs, results, assertionWithZeroThreshold);
+
+      // Max score will be 0, which equals threshold 0, so winner should pass
+      expect(gradingResults[0].pass).toBe(true); // Winner
+      expect(gradingResults[1].pass).toBe(false); // Loser
+    });
+  });
+});

test/matchers/moderation.test.ts

@@ -0,0 +1,189 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesModeration } from '../../src/matchers/moderation';
+import { OpenAiModerationProvider } from '../../src/providers/openai/moderation';
+import { ReplicateModerationProvider } from '../../src/providers/replicate';
+import { LLAMA_GUARD_REPLICATE_PROVIDER } from '../../src/redteam/constants';
+import { mockProcessEnv } from '../util/utils';
+
+describe('matchesModeration', () => {
+  const mockModerationResponse = {
+    flags: [],
+    tokenUsage: { total: 5, prompt: 2, completion: 3 },
+  };
+  const normalizedTokenUsage = {
+    total: 5,
+    prompt: 2,
+    completion: 3,
+    cached: 0,
+    numRequests: 0,
+    completionDetails: {
+      reasoning: 0,
+      acceptedPrediction: 0,
+      rejectedPrediction: 0,
+    },
+  };
+  let restoreProcessEnv = () => {};
+
+  function setTestEnv(overrides: Record<string, string | undefined> = {}) {
+    restoreProcessEnv();
+    restoreProcessEnv = mockProcessEnv({
+      OPENAI_API_KEY: undefined,
+      REPLICATE_API_KEY: undefined,
+      REPLICATE_API_TOKEN: undefined,
+      ...overrides,
+    });
+  }
+
+  beforeEach(() => {
+    setTestEnv();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    restoreProcessEnv();
+    restoreProcessEnv = () => {};
+  });
+
+  it('should skip moderation when assistant response is empty', async () => {
+    const openAiSpy = vi
+      .spyOn(OpenAiModerationProvider.prototype, 'callModerationApi')
+      .mockResolvedValue(mockModerationResponse);
+
+    const result = await matchesModeration({
+      userPrompt: 'test prompt',
+      assistantResponse: '',
+    });
+
+    expect(result).toEqual({
+      pass: true,
+      score: 1,
+      reason: expect.any(String),
+    });
+    expect(openAiSpy).not.toHaveBeenCalled();
+  });
+
+  it('should use OpenAI when OPENAI_API_KEY is present', async () => {
+    setTestEnv({ OPENAI_API_KEY: 'test-key' });
+    const openAiSpy = vi
+      .spyOn(OpenAiModerationProvider.prototype, 'callModerationApi')
+      .mockResolvedValue(mockModerationResponse);
+
+    await matchesModeration({
+      userPrompt: 'test prompt',
+      assistantResponse: 'test response',
+    });
+
+    expect(openAiSpy).toHaveBeenCalledWith('test prompt', 'test response');
+  });
+
+  it('should propagate token usage returned by moderation provider', async () => {
+    setTestEnv({ OPENAI_API_KEY: 'test-key' });
+    vi.spyOn(OpenAiModerationProvider.prototype, 'callModerationApi').mockResolvedValue(
+      mockModerationResponse,
+    );
+
+    const result = await matchesModeration({
+      userPrompt: 'test prompt',
+      assistantResponse: 'test response',
+    });
+
+    expect(result.tokensUsed).toEqual(normalizedTokenUsage);
+  });
+
+  it('should fall back to Replicate when only REPLICATE_API_KEY is present', async () => {
+    setTestEnv({ REPLICATE_API_KEY: 'test-key' });
+    const replicateSpy = vi
+      .spyOn(ReplicateModerationProvider.prototype, 'callModerationApi')
+      .mockResolvedValue(mockModerationResponse);
+
+    await matchesModeration({
+      userPrompt: 'test prompt',
+      assistantResponse: 'test response',
+    });
+
+    expect(replicateSpy).toHaveBeenCalledWith('test prompt', 'test response');
+  });
+
+  it('should respect provider override in grading config', async () => {
+    setTestEnv({ OPENAI_API_KEY: 'test-key' });
+    const replicateSpy = vi
+      .spyOn(ReplicateModerationProvider.prototype, 'callModerationApi')
+      .mockResolvedValue(mockModerationResponse);
+
+    await matchesModeration(
+      {
+        userPrompt: 'test prompt',
+        assistantResponse: 'test response',
+      },
+      {
+        provider: LLAMA_GUARD_REPLICATE_PROVIDER,
+      },
+    );
+
+    expect(replicateSpy).toHaveBeenCalledWith('test prompt', 'test response');
+  });
+
+  it('should fail when the moderation API returns an error', async () => {
+    setTestEnv({ OPENAI_API_KEY: 'test-key' });
+    vi.spyOn(OpenAiModerationProvider.prototype, 'callModerationApi').mockResolvedValue({
+      error: 'provider unavailable',
+      tokenUsage: mockModerationResponse.tokenUsage,
+    });
+
+    await expect(
+      matchesModeration({
+        userPrompt: 'test prompt',
+        assistantResponse: 'test response',
+      }),
+    ).resolves.toEqual({
+      pass: false,
+      score: 0,
+      reason: 'Moderation API error: provider unavailable',
+      tokensUsed: normalizedTokenUsage,
+      // Tagged so inverse-aware callers (not-moderation) don't flip a transport
+      // error into a spurious pass.
+      metadata: { graderError: true },
+    });
+  });
+
+  it('should fail when moderation flags match the requested categories', async () => {
+    setTestEnv({ OPENAI_API_KEY: 'test-key' });
+    vi.spyOn(OpenAiModerationProvider.prototype, 'callModerationApi').mockResolvedValue({
+      flags: [
+        { code: 'violence', description: 'Violence', confidence: 1 },
+        { code: 'hate', description: 'Hate', confidence: 1 },
+      ],
+    });
+
+    await expect(
+      matchesModeration({
+        userPrompt: 'test prompt',
+        assistantResponse: 'test response',
+        categories: ['hate'],
+      }),
+    ).resolves.toEqual({
+      pass: false,
+      score: 0,
+      reason: 'Moderation flags detected: Hate',
+    });
+  });
+
+  it('should pass when flags do not match the requested categories', async () => {
+    setTestEnv({ OPENAI_API_KEY: 'test-key' });
+    vi.spyOn(OpenAiModerationProvider.prototype, 'callModerationApi').mockResolvedValue({
+      flags: [{ code: 'violence', description: 'Violence', confidence: 1 }],
+    });
+
+    await expect(
+      matchesModeration({
+        userPrompt: 'test prompt',
+        assistantResponse: 'test response',
+        categories: ['hate'],
+      }),
+    ).resolves.toEqual({
+      pass: true,
+      score: 1,
+      reason: 'No relevant moderation flags detected',
+    });
+  });
+});

test/matchers/search-rubric.test.ts

@@ -0,0 +1,176 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import type { ApiProvider, ProviderResponse } from '../../src/types/index';
+
+const mocks = vi.hoisted(() => {
+  const configuredProvider: ApiProvider = {
+    id: () => 'openai:gpt-4o',
+    config: {},
+    callApi: vi.fn(async (): Promise<ProviderResponse> => ({ output: 'should not be used' })),
+  };
+  const webSearchProvider = {
+    id: () => 'openai:gpt-5.5-2026-04-23',
+    constructor: { name: 'OpenAiResponsesProvider' },
+    config: {
+      tools: [{ type: 'web_search_preview' }],
+    },
+    callApi: vi.fn(
+      async (): Promise<ProviderResponse> => ({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'web search ok' }),
+        tokenUsage: { total: 5, prompt: 3, completion: 2 },
+      }),
+    ),
+  } as unknown as ApiProvider;
+
+  return {
+    configuredProvider,
+    loadApiProvider: vi.fn(),
+    getDefaultProviders: vi.fn(),
+    webSearchProvider,
+  };
+});
+
+vi.mock('../../src/providers/index', () => ({
+  loadApiProvider: mocks.loadApiProvider,
+}));
+
+vi.mock('../../src/providers/defaults', () => ({
+  getDefaultProviders: mocks.getDefaultProviders,
+}));
+
+describe('matchesSearchRubric', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    mocks.configuredProvider.callApi = vi.fn(
+      async (): Promise<ProviderResponse> => ({ output: 'should not be used' }),
+    ) as ApiProvider['callApi'];
+    mocks.webSearchProvider.callApi = vi.fn(
+      async (): Promise<ProviderResponse> => ({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'web search ok' }),
+        tokenUsage: { total: 5, prompt: 3, completion: 2 },
+      }),
+    ) as ApiProvider['callApi'];
+    mocks.loadApiProvider.mockResolvedValue(mocks.configuredProvider);
+    mocks.getDefaultProviders.mockResolvedValue({
+      webSearchProvider: mocks.webSearchProvider,
+      llmRubricProvider: null,
+      gradingProvider: null,
+    });
+  });
+
+  it('resolves string grading providers before checking web search capability', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+
+    const result = await matchesSearchRubric('Confirm current facts', 'output', {
+      provider: 'openai:responses:gpt-5.5-2026-04-23',
+    });
+
+    expect(mocks.loadApiProvider).toHaveBeenCalled();
+    expect(mocks.loadApiProvider.mock.calls[0][0]).toBe('openai:responses:gpt-5.5-2026-04-23');
+    expect(mocks.configuredProvider.callApi).not.toHaveBeenCalled();
+    expect(mocks.webSearchProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(result).toEqual(
+      expect.objectContaining({
+        pass: true,
+        score: 1,
+        reason: 'web search ok',
+      }),
+    );
+  });
+
+  it('throws when grading config is missing', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+
+    await expect(matchesSearchRubric('Confirm current facts', 'output')).rejects.toThrow(
+      'Cannot grade output without grading config',
+    );
+  });
+
+  it('uses configured providers that already support web search', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+    mocks.loadApiProvider.mockResolvedValue(mocks.webSearchProvider);
+
+    const result = await matchesSearchRubric('Confirm current facts', 'output', {
+      provider: 'openai:responses:gpt-5.5-2026-04-23',
+    });
+
+    expect(mocks.webSearchProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(result.metadata).toEqual({
+      searchProvider: 'openai:gpt-5.5-2026-04-23',
+      searchResults: [],
+    });
+  });
+
+  it('applies assertion thresholds to parsed search results', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+    mocks.webSearchProvider.callApi = vi.fn(
+      async (): Promise<ProviderResponse> => ({
+        output: JSON.stringify({ pass: true, score: 0.4, reason: 'too weak' }),
+      }),
+    ) as ApiProvider['callApi'];
+
+    await expect(
+      matchesSearchRubric(
+        'Confirm current facts',
+        'output',
+        {},
+        {},
+        { type: 'search-rubric', value: 'Confirm current facts', threshold: 0.5 },
+      ),
+    ).resolves.toEqual(
+      expect.objectContaining({
+        pass: false,
+        score: 0.4,
+        reason: 'too weak',
+      }),
+    );
+  });
+
+  it('returns a failure when the search provider returns no output', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+    mocks.webSearchProvider.callApi = vi.fn(
+      async (): Promise<ProviderResponse> => ({
+        error: 'search unavailable',
+      }),
+    ) as ApiProvider['callApi'];
+
+    await expect(matchesSearchRubric('Confirm current facts', 'output', {})).resolves.toEqual(
+      expect.objectContaining({
+        pass: false,
+        score: 0,
+        reason: 'Search rubric evaluation failed: search unavailable',
+      }),
+    );
+  });
+
+  it('falls back to simple pass parsing when JSON extraction fails', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+    mocks.webSearchProvider.callApi = vi.fn(
+      async (): Promise<ProviderResponse> => ({
+        output: 'verdict includes "pass": true',
+      }),
+    ) as ApiProvider['callApi'];
+
+    await expect(matchesSearchRubric('Confirm current facts', 'output', {})).resolves.toEqual(
+      expect.objectContaining({
+        pass: true,
+        score: 1,
+        reason: 'verdict includes "pass": true',
+      }),
+    );
+  });
+
+  it('throws when no web search provider can be resolved', async () => {
+    const { matchesSearchRubric } = await import('../../src/matchers/search');
+    mocks.getDefaultProviders.mockResolvedValue({
+      webSearchProvider: null,
+      llmRubricProvider: null,
+      gradingProvider: mocks.configuredProvider,
+    });
+    mocks.loadApiProvider.mockResolvedValue(null);
+
+    await expect(matchesSearchRubric('Confirm current facts', 'output', {})).rejects.toThrow(
+      'anthropic:messages:claude-sonnet-4-6',
+    );
+  });
+});

test/matchers/shared.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it } from 'vitest';
+import { splitIntoSentences, splitTextIntoSentences } from '../../src/matchers/shared';
+
+describe('splitIntoSentences', () => {
+  it('splits on newlines and drops blank lines', () => {
+    expect(splitIntoSentences('a\n\nb\n  \nc')).toEqual(['a', 'b', 'c']);
+  });
+
+  it('treats a single prose paragraph as one unit (newline-only behavior)', () => {
+    expect(splitIntoSentences('One. Two. Three.')).toEqual(['One. Two. Three.']);
+  });
+
+  it('does not sentence-split a numbered list (markers stay attached to their line)', () => {
+    expect(splitIntoSentences('1. Paris is the capital.\n2. France is in Europe.')).toEqual([
+      '1. Paris is the capital.',
+      '2. France is in Europe.',
+    ]);
+  });
+});
+
+describe('splitTextIntoSentences', () => {
+  it('segments a single prose paragraph on sentence boundaries', () => {
+    expect(
+      splitTextIntoSentences('Paris is the capital of France. France is in Europe. Nice weather.'),
+    ).toEqual(['Paris is the capital of France.', 'France is in Europe.', 'Nice weather.']);
+  });
+
+  it('is unaffected by an incidental leading/trailing newline (regression for the prose fix)', () => {
+    const expected = ['Paris is the capital of France.', 'France is in Europe.'];
+    expect(
+      splitTextIntoSentences('Paris is the capital of France. France is in Europe.\n'),
+    ).toEqual(expected);
+    expect(
+      splitTextIntoSentences('\nParis is the capital of France. France is in Europe.'),
+    ).toEqual(expected);
+    expect(
+      splitTextIntoSentences('Paris is the capital of France. France is in Europe.\r\n'),
+    ).toEqual(expected);
+  });
+
+  it('splits ! and ? boundaries', () => {
+    expect(splitTextIntoSentences('Really? Yes! Absolutely.')).toEqual([
+      'Really?',
+      'Yes!',
+      'Absolutely.',
+    ]);
+  });
+
+  it('trims leading/trailing whitespace on a single-line prose input before splitting', () => {
+    expect(
+      splitTextIntoSentences('   Paris is the capital of France. France is in Europe.   '),
+    ).toEqual(['Paris is the capital of France.', 'France is in Europe.']);
+  });
+
+  it('does not split decimals', () => {
+    expect(splitTextIntoSentences('Pi is about 3.14 in value. It is irrational.')).toEqual([
+      'Pi is about 3.14 in value.',
+      'It is irrational.',
+    ]);
+  });
+
+  it('does not split when a decimal appears at the end of text', () => {
+    expect(splitTextIntoSentences('The value is 3.14.')).toEqual(['The value is 3.14.']);
+  });
+
+  it('treats text with two or more non-empty lines as pre-segmented (one unit per line)', () => {
+    // No abbreviation mis-splits ("i.e.") and no collapsing of multi-sentence lines.
+    expect(splitTextIntoSentences('All employees i.e. engineers.\nThey get leave.')).toEqual([
+      'All employees i.e. engineers.',
+      'They get leave.',
+    ]);
+    expect(splitTextIntoSentences('Line one.\n\n\nLine two.')).toEqual(['Line one.', 'Line two.']);
+  });
+
+  it('drops bare enumeration markers stranded from an inline numbered list', () => {
+    // Sentence-splitting "1. Paris ... 2. France ..." strands the "1." / "2."
+    // markers as their own segments; counting them would inflate sentence-level
+    // metrics (e.g. the RAGAS context-relevance numerator). Only real units remain.
+    expect(splitTextIntoSentences('1. Paris is the capital. 2. France is in Europe.')).toEqual([
+      'Paris is the capital.',
+      'France is in Europe.',
+    ]);
+    // A `)`-style marker on a sentence boundary is likewise dropped when stranded.
+    expect(splitTextIntoSentences('1. First fact. 2. Second fact.')).toEqual([
+      'First fact.',
+      'Second fact.',
+    ]);
+  });
+
+  it('keeps numbers that are part of a sentence, not bare markers', () => {
+    // A decimal or a number embedded in prose is content, never a stray marker.
+    expect(splitTextIntoSentences('Pi is 3.14 here. There are 42 items.')).toEqual([
+      'Pi is 3.14 here.',
+      'There are 42 items.',
+    ]);
+  });
+
+  it('returns an empty array for empty or whitespace-only text', () => {
+    expect(splitTextIntoSentences('')).toEqual([]);
+    expect(splitTextIntoSentences('   \n  \n ')).toEqual([]);
+  });
+});

test/matchers/similarity.test.ts

@@ -0,0 +1,493 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import cliState from '../../src/cliState';
+import { matchesSimilarity } from '../../src/matchers/similarity';
+import { DefaultEmbeddingProvider } from '../../src/providers/openai/defaults';
+import { OpenAiEmbeddingProvider } from '../../src/providers/openai/embedding';
+import * as remoteGeneration from '../../src/redteam/remoteGeneration';
+import * as remoteGrading from '../../src/remoteGrading';
+import { createMockProvider } from '../factories/provider';
+import { mockProcessEnv } from '../util/utils';
+
+import type { OpenAiChatCompletionProvider } from '../../src/providers/openai/chat';
+import type { GradingConfig } from '../../src/types/index';
+
+describe('matchesSimilarity', () => {
+  beforeEach(() => {
+    cliState.config = {};
+    cliState.selectedProviderConfigs = undefined;
+    vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockImplementation((text) => {
+      if (text === 'Expected output' || text === 'Sample output') {
+        return Promise.resolve({
+          embedding: [1, 0, 0],
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        });
+      } else if (text === 'Different output') {
+        return Promise.resolve({
+          embedding: [0, 1, 0],
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        });
+      }
+      return Promise.reject(new Error('Unexpected input'));
+    });
+  });
+
+  afterEach(() => {
+    cliState.selectedProviderConfigs = undefined;
+    vi.restoreAllMocks();
+  });
+
+  it('should pass when similarity is above the threshold', async () => {
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const threshold = 0.5;
+
+    await expect(matchesSimilarity(expected, output, threshold)).resolves.toEqual({
+      pass: true,
+      reason: 'Similarity 1.00 is greater than or equal to threshold 0.5',
+      score: 1,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should fail when similarity is below the threshold', async () => {
+    const expected = 'Expected output';
+    const output = 'Different output';
+    const threshold = 0.9;
+
+    await expect(matchesSimilarity(expected, output, threshold)).resolves.toEqual({
+      pass: false,
+      reason: 'Similarity 0.00 is less than threshold 0.9',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should return zero similarity for zero-magnitude embeddings', async () => {
+    vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockResolvedValue({
+      embedding: [0, 0, 0],
+      tokenUsage: { total: 5, prompt: 2, completion: 3 },
+    });
+
+    await expect(matchesSimilarity('Expected output', 'Sample output', 0.5)).resolves.toEqual({
+      pass: false,
+      reason: 'Similarity 0.00 is less than threshold 0.5',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should include Cloud target context in remote similarity requests', async () => {
+    (cliState as any).config = {
+      providers: ['promptfoo://provider/cloud-target-123'],
+      redteam: {},
+    };
+    vi.spyOn(remoteGeneration, 'shouldGenerateRemote').mockReturnValue(true);
+    vi.spyOn(remoteGrading, 'doRemoteGrading').mockResolvedValue({
+      pass: true,
+      score: 1,
+      reason: 'remote',
+    });
+
+    await matchesSimilarity('Expected output', 'Sample output', 0.5);
+
+    expect(remoteGrading.doRemoteGrading).toHaveBeenCalledWith({
+      task: 'similar',
+      expected: 'Expected output',
+      output: 'Sample output',
+      threshold: 0.5,
+      inverse: false,
+      targetId: 'cloud-target-123',
+    });
+  });
+
+  it('should prefer filtered providers when building remote similarity context', async () => {
+    cliState.config = {
+      providers: ['promptfoo://provider/excluded-target'],
+      redteam: {},
+    };
+    cliState.selectedProviderConfigs = ['promptfoo://provider/selected-target'];
+    vi.spyOn(remoteGeneration, 'shouldGenerateRemote').mockReturnValue(true);
+    vi.spyOn(remoteGrading, 'doRemoteGrading').mockResolvedValue({
+      pass: true,
+      score: 1,
+      reason: 'remote',
+    });
+
+    await matchesSimilarity('Expected output', 'Sample output', 0.5);
+
+    expect(remoteGrading.doRemoteGrading).toHaveBeenCalledWith({
+      task: 'similar',
+      expected: 'Expected output',
+      output: 'Sample output',
+      threshold: 0.5,
+      inverse: false,
+      targetId: 'selected-target',
+    });
+  });
+
+  it('should fail when inverted similarity is above the threshold', async () => {
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const threshold = 0.5;
+
+    await expect(
+      matchesSimilarity(expected, output, threshold, true /* invert */),
+    ).resolves.toEqual({
+      pass: false,
+      reason: 'Similarity 1.00 is greater than or equal to threshold 0.5',
+      score: 0,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should pass when inverted similarity is below the threshold', async () => {
+    const expected = 'Expected output';
+    const output = 'Different output';
+    const threshold = 0.9;
+
+    await expect(
+      matchesSimilarity(expected, output, threshold, true /* invert */),
+    ).resolves.toEqual({
+      pass: true,
+      reason: 'Similarity 0.00 is less than threshold 0.9',
+      score: 1,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+  });
+
+  it('should use the overridden similarity grading config', async () => {
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const threshold = 0.5;
+    const grading: GradingConfig = {
+      provider: {
+        id: 'openai:embedding:text-embedding-ada-9999999',
+        config: {
+          apiKey: 'abc123',
+          temperature: 3.1415926,
+        },
+      },
+    };
+
+    const mockCallApi = vi.spyOn(OpenAiEmbeddingProvider.prototype, 'callEmbeddingApi');
+    mockCallApi.mockImplementation(function (this: OpenAiChatCompletionProvider) {
+      expect(this.config.temperature).toBe(3.1415926);
+      expect(this.getApiKey()).toBe('abc123');
+      return Promise.resolve({
+        embedding: [1, 0, 0],
+        tokenUsage: { total: 5, prompt: 2, completion: 3 },
+      });
+    });
+
+    await expect(matchesSimilarity(expected, output, threshold, false, grading)).resolves.toEqual({
+      pass: true,
+      reason: 'Similarity 1.00 is greater than or equal to threshold 0.5',
+      score: 1,
+      tokensUsed: {
+        total: expect.any(Number),
+        prompt: expect.any(Number),
+        completion: expect.any(Number),
+        cached: expect.any(Number),
+        completionDetails: expect.any(Object),
+        numRequests: 0,
+      },
+    });
+    expect(mockCallApi).toHaveBeenCalledWith('Expected output');
+
+    mockCallApi.mockRestore();
+  });
+
+  it('should throw an error when API call fails', async () => {
+    const expected = 'Expected output';
+    const output = 'Sample output';
+    const threshold = 0.5;
+    const grading: GradingConfig = {
+      provider: {
+        id: 'openai:embedding:text-embedding-ada-9999999',
+        config: {
+          apiKey: 'abc123',
+          temperature: 3.1415926,
+        },
+      },
+    };
+
+    vi.spyOn(OpenAiEmbeddingProvider.prototype, 'callEmbeddingApi').mockRejectedValueOnce(
+      new Error('API call failed'),
+    );
+
+    await expect(async () => {
+      await matchesSimilarity(expected, output, threshold, false, grading);
+    }).rejects.toThrow('API call failed');
+  });
+
+  it('should use Nunjucks templating when PROMPTFOO_DISABLE_TEMPLATING is set', async () => {
+    const restoreEnv = mockProcessEnv({ PROMPTFOO_DISABLE_TEMPLATING: 'true' });
+    try {
+      const expected = 'Expected {{ var }}';
+      const output = 'Output {{ var }}';
+      const threshold = 0.8;
+      const grading: GradingConfig = {
+        provider: DefaultEmbeddingProvider,
+      };
+
+      vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockResolvedValue({
+        embedding: [1, 2, 3],
+        tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      });
+
+      await matchesSimilarity(expected, output, threshold, false, grading);
+
+      expect(DefaultEmbeddingProvider.callEmbeddingApi).toHaveBeenCalledWith('Expected {{ var }}');
+      expect(DefaultEmbeddingProvider.callEmbeddingApi).toHaveBeenCalledWith('Output {{ var }}');
+    } finally {
+      restoreEnv();
+    }
+  });
+
+  describe('dot_product metric', () => {
+    it('should pass when dot product is above threshold', async () => {
+      const expected = 'Expected output';
+      const output = 'Sample output';
+      const threshold = 0.5;
+
+      await expect(
+        matchesSimilarity(expected, output, threshold, false, undefined, 'dot_product'),
+      ).resolves.toMatchObject({
+        pass: true,
+        score: 1,
+      });
+    });
+
+    it('should fail when dot product is below threshold', async () => {
+      const expected = 'Expected output';
+      const output = 'Different output';
+      const threshold = 0.9;
+
+      await expect(
+        matchesSimilarity(expected, output, threshold, false, undefined, 'dot_product'),
+      ).resolves.toMatchObject({
+        pass: false,
+        score: 0,
+      });
+    });
+
+    it('should handle inverse correctly for dot product', async () => {
+      const expected = 'Expected output';
+      const output = 'Sample output';
+      const threshold = 0.5;
+
+      await expect(
+        matchesSimilarity(expected, output, threshold, true, undefined, 'dot_product'),
+      ).resolves.toMatchObject({
+        pass: false,
+        score: 0,
+      });
+    });
+  });
+
+  describe('euclidean metric', () => {
+    beforeEach(() => {
+      vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockImplementation((text) => {
+        if (text === 'Expected output' || text === 'Sample output') {
+          return Promise.resolve({
+            embedding: [1, 0, 0],
+            tokenUsage: { total: 5, prompt: 2, completion: 3 },
+          });
+        } else if (text === 'Different output') {
+          return Promise.resolve({
+            embedding: [0, 1, 0],
+            tokenUsage: { total: 5, prompt: 2, completion: 3 },
+          });
+        }
+        return Promise.reject(new Error('Unexpected input'));
+      });
+    });
+
+    it('should pass when euclidean distance is below threshold', async () => {
+      const expected = 'Expected output';
+      const output = 'Sample output';
+      const threshold = 0.1; // Very low distance = similar
+
+      await expect(
+        matchesSimilarity(expected, output, threshold, false, undefined, 'euclidean'),
+      ).resolves.toMatchObject({
+        pass: true,
+        reason: expect.stringContaining('Distance 0.00 is less than or equal to threshold 0.1'),
+      });
+    });
+
+    it('should fail when euclidean distance is above threshold', async () => {
+      const expected = 'Expected output';
+      const output = 'Different output';
+      const threshold = 0.5; // Distance is ~1.41, above threshold
+
+      await expect(
+        matchesSimilarity(expected, output, threshold, false, undefined, 'euclidean'),
+      ).resolves.toMatchObject({
+        pass: false,
+        reason: expect.stringContaining('Distance 1.41 is greater than threshold 0.5'),
+      });
+    });
+
+    it('should handle inverse correctly for euclidean', async () => {
+      const expected = 'Expected output';
+      const output = 'Different output';
+      const threshold = 0.5;
+
+      // With inverse, we want distance > threshold, which is true here
+      await expect(
+        matchesSimilarity(expected, output, threshold, true, undefined, 'euclidean'),
+      ).resolves.toMatchObject({
+        pass: true,
+        reason: expect.stringContaining('Distance 1.41 is greater than threshold 0.5'),
+      });
+    });
+
+    it('should convert euclidean distance to normalized score', async () => {
+      const expected = 'Expected output';
+      const output = 'Sample output';
+      const threshold = 0.1;
+
+      const result = await matchesSimilarity(
+        expected,
+        output,
+        threshold,
+        false,
+        undefined,
+        'euclidean',
+      );
+
+      // Distance = 0, so score should be 1 / (1 + 0) = 1
+      expect(result.score).toBeCloseTo(1, 2);
+    });
+  });
+
+  describe('metric validation', () => {
+    it('should normalize missing completion details for native similarity providers', async () => {
+      const mockProvider = Object.assign(createMockProvider({ id: 'test-similarity-provider' }), {
+        callSimilarityApi: vi.fn().mockResolvedValue({
+          similarity: 0.9,
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        }),
+      });
+
+      const grading: GradingConfig = {
+        provider: mockProvider as any,
+      };
+
+      await expect(matchesSimilarity('expected', 'output', 0.8, false, grading)).resolves.toEqual(
+        expect.objectContaining({
+          tokensUsed: expect.objectContaining({
+            completionDetails: {
+              reasoning: 0,
+              acceptedPrediction: 0,
+              rejectedPrediction: 0,
+            },
+          }),
+        }),
+      );
+    });
+
+    it('should reject non-cosine metric for callSimilarityApi providers', async () => {
+      const mockProvider = Object.assign(createMockProvider({ id: 'test-similarity-provider' }), {
+        callSimilarityApi: vi.fn().mockResolvedValue({
+          similarity: 0.9,
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        }),
+      });
+
+      const grading: GradingConfig = {
+        provider: mockProvider as any,
+      };
+
+      await expect(
+        matchesSimilarity('expected', 'output', 0.8, false, grading, 'dot_product'),
+      ).resolves.toMatchObject({
+        pass: false,
+        reason: expect.stringContaining('only supports cosine similarity'),
+      });
+    });
+
+    it('should use embeddings for non-cosine metrics when provider supports both APIs', async () => {
+      const mockProvider = Object.assign(createMockProvider({ id: 'hybrid-similarity-provider' }), {
+        callSimilarityApi: vi.fn().mockResolvedValue({
+          similarity: 0.1,
+          tokenUsage: { total: 5, prompt: 2, completion: 3 },
+        }),
+        callEmbeddingApi: vi.fn().mockImplementation((text: string) =>
+          Promise.resolve({
+            embedding: text === 'expected' ? [1, 0] : [0.5, 0],
+            tokenUsage: { total: 5, prompt: 2, completion: 3 },
+          }),
+        ),
+      });
+
+      const grading: GradingConfig = {
+        provider: mockProvider as any,
+      };
+
+      await expect(
+        matchesSimilarity('expected', 'output', 0.4, false, grading, 'dot_product'),
+      ).resolves.toMatchObject({
+        pass: true,
+        score: 0.5,
+      });
+      expect(mockProvider.callSimilarityApi).not.toHaveBeenCalled();
+      expect(mockProvider.callEmbeddingApi).toHaveBeenCalledTimes(2);
+    });
+
+    it('should keep non-cosine metrics local when remote grading is enabled', async () => {
+      (cliState as any).config = { redteam: {} };
+      vi.spyOn(remoteGeneration, 'shouldGenerateRemote').mockReturnValue(true);
+      vi.spyOn(remoteGrading, 'doRemoteGrading').mockResolvedValue({
+        pass: true,
+        score: 1,
+        reason: 'remote',
+      });
+
+      await expect(
+        matchesSimilarity('Expected output', 'Sample output', 0.5, false, undefined, 'dot_product'),
+      ).resolves.toMatchObject({
+        pass: true,
+        score: 1,
+      });
+      expect(remoteGeneration.shouldGenerateRemote).not.toHaveBeenCalled();
+      expect(remoteGrading.doRemoteGrading).not.toHaveBeenCalled();
+    });
+  });
+});

test/matchers/token-tracking.test.ts

@@ -0,0 +1,169 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesGEval, matchesLlmRubric } from '../../src/matchers/llmGrading';
+import { matchesAnswerRelevance } from '../../src/matchers/rag';
+import { loadApiProvider } from '../../src/providers/index';
+import { DefaultGradingProvider } from '../../src/providers/openai/defaults';
+
+describe('Matcher Token Tracking', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.resetAllMocks();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe('matchesLlmRubric', () => {
+    it('should track numRequests in token usage', async () => {
+      const mockProvider = await loadApiProvider('echo');
+      const mockCallApi = vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          pass: true,
+          score: 1,
+          reason: 'Test passed',
+        }),
+        tokenUsage: {
+          total: 100,
+          prompt: 60,
+          completion: 40,
+          cached: 0,
+          numRequests: 1,
+        },
+      });
+
+      mockProvider.callApi = mockCallApi;
+
+      const result = await matchesLlmRubric(
+        'Test rubric',
+        'Test output',
+        { provider: mockProvider },
+        {},
+        undefined,
+      );
+
+      expect(result.tokensUsed).toBeDefined();
+      expect(result.tokensUsed?.numRequests).toBe(1);
+      expect(result.tokensUsed?.total).toBe(100);
+      expect(result.tokensUsed?.prompt).toBe(60);
+      expect(result.tokensUsed?.completion).toBe(40);
+      expect(result.tokensUsed?.cached).toBe(0);
+    });
+
+    it('should default numRequests to 0 when not provided', async () => {
+      const mockProvider = await loadApiProvider('echo');
+      const mockCallApi = vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          pass: true,
+          score: 1,
+          reason: 'Test passed',
+        }),
+        tokenUsage: {
+          total: 100,
+          prompt: 60,
+          completion: 40,
+          cached: 0,
+          // numRequests not provided
+        },
+      });
+
+      mockProvider.callApi = mockCallApi;
+
+      const result = await matchesLlmRubric(
+        'Test rubric',
+        'Test output',
+        { provider: mockProvider },
+        {},
+        undefined,
+      );
+
+      expect(result.tokensUsed).toBeDefined();
+      expect(result.tokensUsed?.numRequests).toBe(0);
+    });
+  });
+
+  describe('matchesGEval', () => {
+    it('should accumulate numRequests across multiple API calls', async () => {
+      const mockCallApi = vi.spyOn(DefaultGradingProvider, 'callApi');
+
+      // First call for steps
+      mockCallApi.mockResolvedValueOnce({
+        output: JSON.stringify({
+          steps: ['Step 1', 'Step 2'],
+        }),
+        tokenUsage: {
+          total: 50,
+          prompt: 30,
+          completion: 20,
+          cached: 0,
+          numRequests: 1,
+        },
+      });
+
+      // Second call for evaluation
+      mockCallApi.mockResolvedValueOnce({
+        output: JSON.stringify({
+          score: 8,
+          reason: 'Good response',
+        }),
+        tokenUsage: {
+          total: 100,
+          prompt: 60,
+          completion: 40,
+          cached: 10,
+          numRequests: 1,
+        },
+      });
+
+      const result = await matchesGEval('Test criteria', 'Test input', 'Test output', 0.7, {});
+
+      expect(result.tokensUsed).toBeDefined();
+      expect(result.tokensUsed?.numRequests).toBe(2); // 1 from steps + 1 from evaluation
+      expect(result.tokensUsed?.total).toBe(150); // 50 + 100
+      expect(result.tokensUsed?.prompt).toBe(90); // 30 + 60
+      expect(result.tokensUsed?.completion).toBe(60); // 20 + 40
+      expect(result.tokensUsed?.cached).toBe(10); // 0 + 10
+    });
+  });
+
+  describe('matchesAnswerRelevance', () => {
+    it('should track numRequests from embedding calls', async () => {
+      const { DefaultGradingProvider, DefaultEmbeddingProvider } = await import(
+        '../../src/providers/openai/defaults'
+      );
+
+      const mockCallApi = vi.spyOn(DefaultGradingProvider, 'callApi');
+      const mockCallEmbeddingApi = vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi');
+
+      // Mock text generation calls (3 times for candidate questions)
+      mockCallApi.mockResolvedValue({
+        output: 'Generated question',
+        tokenUsage: {
+          total: 20,
+          prompt: 10,
+          completion: 10,
+          cached: 0,
+          numRequests: 1,
+        },
+      });
+
+      // Mock embedding calls
+      mockCallEmbeddingApi.mockResolvedValue({
+        embedding: [1, 0, 0],
+        tokenUsage: {
+          total: 5,
+          prompt: 5,
+          completion: 0,
+          cached: 0,
+          numRequests: 1,
+        },
+      });
+
+      const result = await matchesAnswerRelevance('Test input', 'Test output', 0.7, {});
+
+      expect(result.tokensUsed).toBeDefined();
+      // 3 text generation + 1 input embedding + 3 question embeddings = 7 requests
+      expect(result.tokensUsed?.numRequests).toBe(7);
+    });
+  });
+});

test/matchers/trajectory-goal-success.test.ts

@@ -0,0 +1,160 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesTrajectoryGoalSuccess } from '../../src/matchers/llmGrading';
+import { createMockProvider, createProviderResponse } from '../factories/provider';
+
+import type { Assertion, GradingConfig } from '../../src/types/index';
+
+describe('matchesTrajectoryGoalSuccess', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('renders the goal, trajectory, and output into the grading prompt', async () => {
+    const provider = createMockProvider({
+      response: createProviderResponse({
+        output: JSON.stringify({ pass: true, score: 0.85, reason: 'Goal achieved' }),
+        tokenUsage: { total: 9, prompt: 5, completion: 4 },
+      }),
+    });
+
+    const grading: GradingConfig = {
+      provider,
+      rubricPrompt: 'Goal={{ goal }}\nTrajectory={{ trajectory }}\nOutput={{ output }}',
+    };
+
+    const result = await matchesTrajectoryGoalSuccess(
+      'Resolve the order lookup task',
+      '{"stepCount":2}',
+      'The order shipped yesterday.',
+      grading,
+      { orderId: '123' },
+    );
+
+    expect(provider.callApi).toHaveBeenCalledWith(
+      expect.stringContaining('Goal=Resolve the order lookup task'),
+      expect.objectContaining({
+        prompt: expect.objectContaining({
+          label: 'trajectory:goal-success',
+          raw: expect.stringContaining('Trajectory={"stepCount":2}'),
+        }),
+        vars: expect.objectContaining({
+          goal: 'Resolve the order lookup task',
+          orderId: '123',
+          output: 'The order shipped yesterday.',
+          trajectory: '{"stepCount":2}',
+        }),
+      }),
+    );
+
+    expect(result).toEqual({
+      pass: true,
+      score: 0.85,
+      reason: 'Goal achieved',
+      tokensUsed: {
+        total: 9,
+        prompt: 5,
+        completion: 4,
+        cached: 0,
+        completionDetails: {
+          reasoning: 0,
+          acceptedPrediction: 0,
+          rejectedPrediction: 0,
+        },
+        numRequests: 0,
+      },
+      metadata: {
+        renderedGradingPrompt:
+          'Goal=Resolve the order lookup task\nTrajectory={"stepCount":2}\nOutput=The order shipped yesterday.',
+      },
+      assertion: undefined,
+    });
+  });
+
+  it('throws when grading config is not provided', async () => {
+    await expect(
+      matchesTrajectoryGoalSuccess('Resolve the task', '{"stepCount":1}', 'output', undefined),
+    ).rejects.toThrow('Cannot grade output without grading config');
+  });
+
+  it('applies assertion thresholds to the returned score', async () => {
+    const provider = createMockProvider({
+      response: createProviderResponse({
+        output: JSON.stringify({ pass: true, score: 0.6, reason: 'Partial progress' }),
+        tokenUsage: { total: 4, prompt: 2, completion: 2 },
+      }),
+    });
+
+    const grading: GradingConfig = {
+      provider,
+      rubricPrompt: 'Goal={{ goal }}',
+    };
+
+    const assertion: Assertion = {
+      type: 'trajectory:goal-success',
+      threshold: 0.8,
+      value: 'Resolve the task',
+    };
+
+    const result = await matchesTrajectoryGoalSuccess(
+      'Resolve the task',
+      '{"stepCount":1}',
+      'I found part of the answer.',
+      grading,
+      {},
+      assertion,
+    );
+
+    expect(result).toEqual(
+      expect.objectContaining({
+        pass: false,
+        score: 0.6,
+        reason: 'Partial progress',
+      }),
+    );
+  });
+
+  it('does not allow caller vars to override goal, trajectory, or output', async () => {
+    const provider = createMockProvider({
+      response: createProviderResponse({
+        output: JSON.stringify({ pass: true, score: 1, reason: 'Goal achieved' }),
+        tokenUsage: { total: 6, prompt: 3, completion: 3 },
+      }),
+    });
+
+    const grading: GradingConfig = {
+      provider,
+      rubricPrompt:
+        'Goal={{ goal }}\nTrajectory={{ trajectory }}\nOutput={{ output }}\nOrder={{ orderId }}',
+    };
+
+    await matchesTrajectoryGoalSuccess(
+      'Resolve the order lookup task',
+      '{"stepCount":2}',
+      'The order shipped yesterday.',
+      grading,
+      {
+        goal: 'spoofed goal',
+        trajectory: 'spoofed trajectory',
+        output: 'spoofed output',
+        orderId: '123',
+      },
+    );
+
+    expect(provider.callApi).toHaveBeenCalledWith(
+      expect.stringContaining('Goal=Resolve the order lookup task'),
+      expect.objectContaining({
+        prompt: expect.objectContaining({
+          raw: expect.stringContaining(
+            'Goal=Resolve the order lookup task\nTrajectory={"stepCount":2}\nOutput=The order shipped yesterday.\nOrder=123',
+          ),
+        }),
+        vars: expect.objectContaining({
+          goal: 'Resolve the order lookup task',
+          trajectory: '{"stepCount":2}',
+          output: 'The order shipped yesterday.',
+          orderId: '123',
+        }),
+      }),
+    );
+  });
+});

test/matchers/utils.test.ts

@@ -0,0 +1,459 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import cliState from '../../src/cliState';
+import {
+  getAndCheckProvider,
+  getGradingProvider,
+  getRemoteGradingContext,
+} from '../../src/matchers/providers';
+import { renderLlmRubricPrompt } from '../../src/matchers/rubric';
+import {
+  DefaultEmbeddingProvider,
+  DefaultGradingProvider,
+} from '../../src/providers/openai/defaults';
+import { createMockProvider } from '../factories/provider';
+import { mockProcessEnv } from '../util/utils';
+
+import type { ProviderTypeMap } from '../../src/types/index';
+
+describe('getRemoteGradingContext', () => {
+  beforeEach(() => {
+    cliState.config = undefined;
+    cliState.selectedProviderConfigs = undefined;
+  });
+
+  afterEach(() => {
+    cliState.config = undefined;
+    cliState.selectedProviderConfigs = undefined;
+  });
+
+  it('prefers the actively selected provider configs', () => {
+    cliState.config = { providers: ['promptfoo://provider/excluded-target'] };
+    cliState.selectedProviderConfigs = ['promptfoo://provider/selected-target'];
+
+    expect(getRemoteGradingContext()).toEqual({ targetId: 'selected-target' });
+  });
+
+  it('falls back to the configured providers', () => {
+    cliState.config = { providers: ['promptfoo://provider/configured-target'] };
+
+    expect(getRemoteGradingContext()).toEqual({ targetId: 'configured-target' });
+  });
+
+  it('does not fall back to configured providers when the filter matched nothing', () => {
+    cliState.config = { providers: ['promptfoo://provider/configured-target'] };
+    cliState.selectedProviderConfigs = [];
+
+    expect(getRemoteGradingContext()).toEqual({});
+  });
+});
+
+describe('getGradingProvider', () => {
+  it('should return the correct provider when provider is a string', async () => {
+    const provider = await getGradingProvider(
+      'text',
+      'openai:chat:gpt-4o-mini-foobar',
+      DefaultGradingProvider,
+    );
+    // ok for this not to match exactly when the string is parsed
+    expect(provider?.id()).toBe('openai:gpt-4o-mini-foobar');
+  });
+
+  it('should return the correct provider when provider is an ApiProvider', async () => {
+    const provider = await getGradingProvider(
+      'embedding',
+      DefaultEmbeddingProvider,
+      DefaultGradingProvider,
+    );
+    expect(provider).toBe(DefaultEmbeddingProvider);
+  });
+
+  it('should return the correct provider when provider is ProviderOptions', async () => {
+    const providerOptions = {
+      id: 'openai:chat:gpt-4o-mini-foobar',
+      config: {
+        apiKey: 'abc123',
+        temperature: 3.1415926,
+      },
+    };
+    const provider = await getGradingProvider('text', providerOptions, DefaultGradingProvider);
+    expect(provider?.id()).toBe('openai:chat:gpt-4o-mini-foobar');
+  });
+
+  it('should return the default provider when provider is not provided', async () => {
+    const provider = await getGradingProvider('text', undefined, DefaultGradingProvider);
+    expect(provider).toBe(DefaultGradingProvider);
+  });
+});
+
+describe('getAndCheckProvider', () => {
+  it('should return the default provider when provider is not defined', async () => {
+    await expect(
+      getAndCheckProvider('text', undefined, DefaultGradingProvider, 'test check'),
+    ).resolves.toBe(DefaultGradingProvider);
+  });
+
+  it('should throw when explicitly configured provider does not support type', async () => {
+    const provider = {
+      id: () => 'test-provider',
+      callApi: () => Promise.resolve({ output: 'test' }),
+    };
+    await expect(
+      getAndCheckProvider('embedding', provider, DefaultEmbeddingProvider, 'test check'),
+    ).rejects.toThrow('is not a valid embedding provider');
+  });
+
+  it('should return the provider if it implements the required method', async () => {
+    const provider = {
+      id: () => 'test-provider',
+      callApi: () => Promise.resolve({ output: 'test' }),
+      callEmbeddingApi: () => Promise.resolve({ embedding: [] }),
+    };
+    const result = await getAndCheckProvider(
+      'embedding',
+      provider,
+      DefaultEmbeddingProvider,
+      'test check',
+    );
+    expect(result).toBe(provider);
+  });
+
+  it('should return the default provider when no provider is specified', async () => {
+    const provider = await getGradingProvider('text', undefined, DefaultGradingProvider);
+    expect(provider).toBe(DefaultGradingProvider);
+  });
+
+  it('should return a specific provider when a provider id is specified', async () => {
+    const provider = await getGradingProvider('text', 'openai:chat:foo', DefaultGradingProvider);
+    // loadApiProvider removes `chat` from the id
+    expect(provider?.id()).toBe('openai:foo');
+  });
+
+  it('should return a provider from ApiProvider when specified', async () => {
+    const providerOptions = createMockProvider({ id: 'custom-provider', response: {} });
+    const provider = await getGradingProvider('text', providerOptions, DefaultGradingProvider);
+    expect(provider?.id()).toBe('custom-provider');
+  });
+
+  it('should return a provider from ProviderTypeMap when specified', async () => {
+    const providerTypeMap: ProviderTypeMap = {
+      text: {
+        id: 'openai:chat:foo',
+      },
+      embedding: {
+        id: 'openai:embedding:bar',
+      },
+    };
+    const provider = await getGradingProvider('text', providerTypeMap, DefaultGradingProvider);
+    expect(provider?.id()).toBe('openai:chat:foo');
+  });
+
+  it('should return a provider from ProviderTypeMap with basic strings', async () => {
+    const providerTypeMap: ProviderTypeMap = {
+      text: 'openai:chat:foo',
+      embedding: 'openai:embedding:bar',
+    };
+    const provider = await getGradingProvider('text', providerTypeMap, DefaultGradingProvider);
+    expect(provider?.id()).toBe('openai:foo');
+  });
+
+  it('should throw an error when the provider does not match the type', async () => {
+    const providerTypeMap: ProviderTypeMap = {
+      embedding: {
+        id: 'openai:embedding:foo',
+      },
+    };
+    await expect(
+      getGradingProvider('text', providerTypeMap, DefaultGradingProvider),
+    ).rejects.toThrow(
+      new Error(
+        `Invalid provider definition for output type 'text': ${JSON.stringify(
+          providerTypeMap,
+          null,
+          2,
+        )}`,
+      ),
+    );
+  });
+});
+
+describe('tryParse and renderLlmRubricPrompt', () => {
+  let tryParse: (content: string | null | undefined) => any;
+
+  beforeAll(async () => {
+    const context: { capturedFn: null | Function } = { capturedFn: null };
+
+    await renderLlmRubricPrompt('{"test":"value"}', {
+      __capture(fn: Function) {
+        context.capturedFn = fn;
+        return 'captured';
+      },
+    });
+
+    tryParse = function (content: string | null | undefined) {
+      try {
+        if (content === null || content === undefined) {
+          return content;
+        }
+        return JSON.parse(content);
+      } catch {}
+      return content;
+    };
+  });
+
+  it('should parse valid JSON', () => {
+    const input = '{"key": "value"}';
+    expect(tryParse(input)).toEqual({ key: 'value' });
+  });
+
+  it('should return original string for invalid JSON', () => {
+    const input = 'not json';
+    expect(tryParse(input)).toBe('not json');
+  });
+
+  it('should handle empty string', () => {
+    const input = '';
+    expect(tryParse(input)).toBe('');
+  });
+
+  it('should handle null and undefined', () => {
+    expect(tryParse(null)).toBeNull();
+    expect(tryParse(undefined)).toBeUndefined();
+  });
+
+  it('should render strings inside JSON objects', async () => {
+    const template = '{"role": "user", "content": "Hello {{name}}"}';
+    const result = await renderLlmRubricPrompt(template, { name: 'World' });
+    const parsed = JSON.parse(result);
+    expect(parsed).toEqual({ role: 'user', content: 'Hello World' });
+  });
+
+  it('should preserve JSON structure while rendering only strings', async () => {
+    const template = '{"nested": {"text": "{{var}}", "number": 42}}';
+    const result = await renderLlmRubricPrompt(template, { var: 'test' });
+    const parsed = JSON.parse(result);
+    expect(parsed).toEqual({ nested: { text: 'test', number: 42 } });
+  });
+
+  it('should handle non-JSON templates with legacy rendering', async () => {
+    const template = 'Hello {{name}}';
+    const result = await renderLlmRubricPrompt(template, { name: 'World' });
+    expect(result).toBe('Hello World');
+  });
+
+  it('should handle complex objects in context', async () => {
+    const template = '{"text": "{{object}}"}';
+    const complexObject = { foo: 'bar', baz: [1, 2, 3] };
+    const result = await renderLlmRubricPrompt(template, { object: complexObject });
+    const parsed = JSON.parse(result);
+    expect(typeof parsed.text).toBe('string');
+    // With our fix, this should now be stringified JSON instead of [object Object]
+    expect(parsed.text).toBe(JSON.stringify(complexObject));
+  });
+
+  it('should properly stringify objects', async () => {
+    const template = 'Source Text:\n{{input}}';
+    // Create objects that would typically cause the [object Object] issue
+    const objects = [
+      { name: 'Object 1', properties: { color: 'red', size: 'large' } },
+      { name: 'Object 2', properties: { color: 'blue', size: 'small' } },
+    ];
+
+    const result = await renderLlmRubricPrompt(template, { input: objects });
+
+    // With our fix, this should properly stringify the objects
+    expect(result).not.toContain('[object Object]');
+    expect(result).toContain(JSON.stringify(objects[0]));
+    expect(result).toContain(JSON.stringify(objects[1]));
+  });
+
+  it('should handle mixed arrays of objects and primitives', async () => {
+    const template = 'Items: {{items}}';
+    const mixedArray = ['string item', { name: 'Object item' }, 42, [1, 2, 3]];
+
+    const result = await renderLlmRubricPrompt(template, { items: mixedArray });
+
+    // Objects in array should be stringified
+    expect(result).not.toContain('[object Object]');
+    expect(result).toContain('string item');
+    expect(result).toContain(JSON.stringify({ name: 'Object item' }));
+    expect(result).toContain('42');
+    expect(result).toContain(JSON.stringify([1, 2, 3]));
+  });
+
+  it('should render arrays of objects correctly', async () => {
+    const template = '{"items": [{"name": "{{name1}}"}, {"name": "{{name2}}"}]}';
+    const result = await renderLlmRubricPrompt(template, { name1: 'Alice', name2: 'Bob' });
+    const parsed = JSON.parse(result);
+    expect(parsed).toEqual({
+      items: [{ name: 'Alice' }, { name: 'Bob' }],
+    });
+  });
+
+  it('should handle multiline strings', async () => {
+    const template = `{"content": "Line 1\\nLine {{number}}\\nLine 3"}`;
+    const result = await renderLlmRubricPrompt(template, { number: '2' });
+    const parsed = JSON.parse(result);
+    expect(parsed).toEqual({
+      content: 'Line 1\nLine 2\nLine 3',
+    });
+  });
+
+  it('should handle nested templates', async () => {
+    const template = '{"outer": "{{value1}}", "inner": {"value": "{{value2}}"}}';
+    const result = await renderLlmRubricPrompt(template, {
+      value1: 'outer value',
+      value2: 'inner value',
+    });
+    const parsed = JSON.parse(result);
+    expect(parsed).toEqual({
+      outer: 'outer value',
+      inner: { value: 'inner value' },
+    });
+  });
+
+  it('should handle escaping in JSON strings', async () => {
+    const template = '{"content": "This needs \\"escaping\\" and {{var}} too"}';
+    const result = await renderLlmRubricPrompt(template, { var: 'var with "quotes"' });
+    const parsed = JSON.parse(result);
+    expect(parsed.content).toBe('This needs "escaping" and var with "quotes" too');
+  });
+
+  it('should work with nested arrays and objects', async () => {
+    const template = JSON.stringify({
+      role: 'system',
+      content: 'Process this: {{input}}',
+      config: {
+        options: [
+          { id: 1, label: '{{option1}}' },
+          { id: 2, label: '{{option2}}' },
+        ],
+      },
+    });
+
+    const evalResult = await renderLlmRubricPrompt(template, {
+      input: 'test input',
+      option1: 'First Option',
+      option2: 'Second Option',
+    });
+
+    const parsed = JSON.parse(evalResult);
+    expect(parsed.content).toBe('Process this: test input');
+    expect(parsed.config.options[0].label).toBe('First Option');
+    expect(parsed.config.options[1].label).toBe('Second Option');
+  });
+
+  it('should handle rendering statements with join filter', async () => {
+    const statements = ['Statement 1', 'Statement 2', 'Statement 3'];
+    const template = 'statements:\n{{statements|join("\\n")}}';
+    const result = await renderLlmRubricPrompt(template, { statements });
+
+    const expected = 'statements:\nStatement 1\nStatement 2\nStatement 3';
+    expect(result).toBe(expected);
+  });
+
+  it('should stringify objects in arrays', async () => {
+    const template = 'Items: {{items}}';
+    const items = [{ name: 'Item 1', price: 10 }, 'string item', { name: 'Item 2', price: 20 }];
+
+    const result = await renderLlmRubricPrompt(template, { items });
+
+    expect(result).not.toContain('[object Object]');
+    expect(result).toContain(JSON.stringify(items[0]));
+    expect(result).toContain('string item');
+    expect(result).toContain(JSON.stringify(items[2]));
+  });
+
+  it('should stringify deeply nested objects and arrays', async () => {
+    const template = 'Complex data: {{data}}';
+    const data = {
+      products: [
+        {
+          name: 'Item 1',
+          price: 10,
+          details: {
+            color: 'red',
+            specs: { weight: '2kg', dimensions: { width: 10, height: 20 } },
+          },
+        },
+        'string item',
+        {
+          name: 'Item 2',
+          price: 20,
+          nested: [{ a: 1 }, { b: 2 }],
+          metadata: { tags: ['electronics', 'gadget'] },
+        },
+      ],
+    };
+
+    const result = await renderLlmRubricPrompt(template, { data });
+
+    expect(result).not.toContain('[object Object]');
+    expect(result).toContain('"specs":{"weight":"2kg"');
+    expect(result).toContain('"dimensions":{"width":10,"height":20}');
+    expect(result).toContain('[{"a":1},{"b":2}]');
+    expect(result).toContain('"tags":["electronics","gadget"]');
+    expect(result).toContain('string item');
+  });
+});
+
+describe('PROMPTFOO_DISABLE_OBJECT_STRINGIFY environment variable', () => {
+  afterEach(() => {
+    // Clean up environment variable after each test
+    mockProcessEnv({ PROMPTFOO_DISABLE_OBJECT_STRINGIFY: undefined });
+  });
+
+  describe('Default behavior (PROMPTFOO_DISABLE_OBJECT_STRINGIFY=false)', () => {
+    beforeEach(() => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_OBJECT_STRINGIFY: 'false' });
+    });
+
+    it('should stringify objects to prevent [object Object] issues', async () => {
+      const template = 'Product: {{product}}';
+      const product = { name: 'Headphones', price: 99.99 };
+
+      const result = await renderLlmRubricPrompt(template, { product });
+
+      expect(result).not.toContain('[object Object]');
+      expect(result).toBe(`Product: ${JSON.stringify(product)}`);
+    });
+
+    it('should stringify objects in arrays', async () => {
+      const template = 'Items: {{items}}';
+      const items = [{ name: 'Item 1', price: 10 }, 'string item', { name: 'Item 2', price: 20 }];
+
+      const result = await renderLlmRubricPrompt(template, { items });
+
+      expect(result).not.toContain('[object Object]');
+      expect(result).toContain(JSON.stringify(items[0]));
+      expect(result).toContain('string item');
+      expect(result).toContain(JSON.stringify(items[2]));
+    });
+  });
+
+  describe('Object access enabled (PROMPTFOO_DISABLE_OBJECT_STRINGIFY=true)', () => {
+    beforeEach(() => {
+      mockProcessEnv({ PROMPTFOO_DISABLE_OBJECT_STRINGIFY: 'true' });
+    });
+
+    it('should allow direct object property access', async () => {
+      const template = 'Product: {{product.name}} - ${{product.price}}';
+      const product = { name: 'Headphones', price: 99.99 };
+
+      const result = await renderLlmRubricPrompt(template, { product });
+
+      expect(result).toBe('Product: Headphones - $99.99');
+    });
+
+    it('should allow array indexing and property access', async () => {
+      const template = 'First item: {{items[0].name}}';
+      const items = [
+        { name: 'First Item', price: 10 },
+        { name: 'Second Item', price: 20 },
+      ];
+
+      const result = await renderLlmRubricPrompt(template, { items });
+
+      expect(result).toBe('First item: First Item');
+    });
+  });
+});