chore: import upstream snapshot with attribution

test/external/assertions.test.ts

@@ -0,0 +1,277 @@
+import { describe, expect, it, vi } from 'vitest';
+import { handleConversationRelevance } from '../../src/external/assertions/deepeval';
+import { createMockProvider as createFactoryProvider } from '../factories/provider';
+
+import type { ApiProvider, AssertionParams, AtomicTestCase } from '../../src/types/index';
+
+const createMockProvider = (output: string, reason?: string) =>
+  createFactoryProvider({
+    response: {
+      output: JSON.stringify({
+        verdict: output,
+        ...(output === 'no' && {
+          reason: reason || 'Response is completely irrelevant to the message input',
+        }),
+      }),
+      tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+    },
+  });
+
+// Mock getAndCheckProvider
+vi.mock('../../src/matchers/providers', async () => {
+  const actual = await vi.importActual('../../src/matchers/providers');
+  return {
+    ...actual,
+    getAndCheckProvider: vi.fn().mockImplementation((_type, provider) => {
+      return provider;
+    }),
+  };
+});
+
+const defaultParams = {
+  baseType: 'conversation-relevance' as const,
+  assertionValueContext: {
+    vars: {},
+    test: {} as AtomicTestCase,
+    prompt: 'test prompt',
+    logProbs: undefined,
+    provider: createMockProvider('yes'),
+    providerResponse: { output: 'hello world' },
+  },
+  output: 'hello world',
+  providerResponse: { output: 'hello world' },
+  test: {
+    vars: {},
+    options: {
+      provider: createMockProvider('yes'),
+    },
+  } as AtomicTestCase,
+  inverse: false,
+};
+
+describe('handleConversationRelevance', () => {
+  it('should handle single message pairs', async () => {
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+      },
+      prompt: 'What is the weather like?',
+      outputString: 'The weather is sunny today.',
+      test: {
+        vars: {},
+        options: {
+          provider: createMockProvider('yes'),
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it('should ignore outputString when _conversation is present and non-empty', async () => {
+    const conversation = [{ input: 'Hi', output: 'Hello! How can I help you?' }];
+
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+      },
+      prompt: 'This should be ignored',
+      outputString: 'This should be ignored',
+      test: {
+        vars: {
+          _conversation: conversation,
+        },
+        options: {
+          provider: createMockProvider('yes'),
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it('should evaluate sliding windows in a conversation', async () => {
+    const conversation = [
+      { input: 'Hi', output: 'Hello! How can I help you?' },
+      { input: 'What is the weather like?', output: 'The weather is sunny today.' },
+      {
+        input: 'What should I wear?',
+        output: 'Given the sunny weather, you might want to wear light clothes.',
+      },
+      { input: 'Thanks!', output: "You're welcome! Enjoy the sunshine!" },
+      { input: 'What is 2+2?', output: 'The capital of France is Paris.' }, // Irrelevant response
+    ];
+
+    // Create a smart mock provider that only fails for the math/Paris response
+    const smartMockProvider = createFactoryProvider({
+      callApi: vi.fn<ApiProvider['callApi']>().mockImplementation(async (prompt) => {
+        // Check if the prompt contains the math question and Paris response
+        const hasIrrelevantResponse =
+          (prompt as string).includes('2+2') && (prompt as string).includes('Paris');
+        return {
+          output: JSON.stringify({
+            verdict: hasIrrelevantResponse ? 'no' : 'yes',
+            ...(hasIrrelevantResponse && {
+              reason:
+                'The response about Paris is completely unrelated to the mathematical question about 2+2',
+            }),
+          }),
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        };
+      }),
+    });
+
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.85, // Score will be 0.8 (4/5), so threshold > 0.8 to fail
+        config: {
+          windowSize: 3,
+        },
+      },
+      outputString: 'This should be ignored',
+      test: {
+        vars: {
+          _conversation: conversation,
+        },
+        options: {
+          provider: smartMockProvider,
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(false);
+    expect(result.score).toBeLessThan(1);
+    expect(result.reason).toContain('The response about Paris is completely unrelated');
+  });
+
+  it('should handle conversations shorter than window size', async () => {
+    const shortConversation = [
+      { input: 'Hi', output: 'Hello! How can I help you?' },
+      { input: 'What is the weather like?', output: 'The weather is sunny today.' },
+    ];
+
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertionValueContext: {
+        ...defaultParams.assertionValueContext,
+        provider: createMockProvider('yes'),
+      },
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+        config: {
+          windowSize: 5,
+        },
+      },
+      outputString: 'This should be ignored',
+      test: {
+        vars: {
+          _conversation: shortConversation,
+        },
+        options: {
+          provider: createMockProvider('yes'),
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it('should use default window size when not specified', async () => {
+    const conversation = [
+      { input: 'Hi', output: 'Hello! How can I help you?' },
+      { input: 'What is the weather like?', output: 'The weather is sunny today.' },
+      {
+        input: 'What should I wear?',
+        output: 'Given the sunny weather, you might want to wear light clothes.',
+      },
+      { input: 'Thanks!', output: "You're welcome! Enjoy the sunshine!" },
+    ];
+
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertionValueContext: {
+        ...defaultParams.assertionValueContext,
+        provider: createMockProvider('yes'),
+      },
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+      },
+      outputString: 'This should be ignored',
+      test: {
+        vars: {
+          _conversation: conversation,
+        },
+        options: {
+          provider: createMockProvider('yes'),
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result).toHaveProperty('score');
+    expect(result).toHaveProperty('pass');
+  });
+
+  it('should use outputString when _conversation is empty', async () => {
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+      },
+      prompt: 'What is the weather like?',
+      outputString: 'The weather is sunny today.',
+      test: {
+        vars: {
+          _conversation: [], // Empty conversation array
+        },
+        options: {
+          provider: createMockProvider('yes'),
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+
+  it('should use outputString when _conversation is undefined', async () => {
+    const params: AssertionParams = {
+      ...defaultParams,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+      },
+      prompt: 'What is the weather like?',
+      outputString: 'The weather is sunny today.',
+      test: {
+        vars: {
+          // _conversation is not defined
+        },
+        options: {
+          provider: createMockProvider('yes'),
+        },
+      },
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+});

test/external/conversationRelevancy.test.ts

@@ -0,0 +1,231 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { handleConversationRelevance } from '../../src/external/assertions/deepeval';
+import { ConversationRelevancyTemplate } from '../../src/external/matchers/conversationRelevancyTemplate';
+import { matchesConversationRelevance } from '../../src/external/matchers/deepeval';
+import { getDefaultProviders } from '../../src/providers/defaults';
+
+import type { AssertionParams, AtomicTestCase } from '../../src/types/index';
+
+vi.mock('../../src/providers/defaults', () => ({
+  getDefaultProviders: vi.fn(),
+}));
+
+vi.mock('../../src/matchers/providers', async () => {
+  const actual = await vi.importActual('../../src/matchers/providers');
+  return {
+    ...actual,
+    getAndCheckProvider: vi.fn().mockImplementation(async (_type, provider, defaultProvider) => {
+      return provider || defaultProvider;
+    }),
+  };
+});
+
+describe('ConversationRelevancyTemplate', () => {
+  describe('generateVerdicts', () => {
+    it('should generate proper verdict prompt', () => {
+      const messages = [
+        { role: 'user' as const, content: 'What is the weather?' },
+        { role: 'assistant' as const, content: 'It is sunny today.' },
+      ];
+
+      const prompt = ConversationRelevancyTemplate.generateVerdicts(messages);
+      expect(prompt).toContain(
+        'generate a JSON object to indicate whether the LAST `assistant` message is relevant',
+      );
+      expect(prompt).toContain(JSON.stringify(messages, null, 2));
+    });
+  });
+
+  describe('generateReason', () => {
+    it('should generate proper reason prompt', () => {
+      const score = 0.6;
+      const irrelevancies = [
+        'Response about weather was not related to math question',
+        'Assistant talked about food when asked about programming',
+      ];
+
+      const prompt = ConversationRelevancyTemplate.generateReason(score, irrelevancies);
+      expect(prompt).toContain(`Relevancy Score:\n${score}`);
+      expect(prompt).toContain(JSON.stringify(irrelevancies, null, 2));
+    });
+  });
+});
+
+describe('matchesConversationRelevance with template', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should use ConversationRelevancyTemplate for prompts', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ verdict: 'yes' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      { input: 'What is 2+2?', output: '4' },
+      { input: 'What is the capital of France?', output: 'Paris' },
+    ];
+
+    await matchesConversationRelevance(messages, 0.5);
+
+    const callArg = mockProvider.callApi.mock.calls[0][0];
+    // Should contain the template structure
+    expect(callArg).toContain(
+      'generate a JSON object to indicate whether the LAST `assistant` message is relevant',
+    );
+    expect(callArg).toContain('"role": "user"');
+    expect(callArg).toContain('"role": "assistant"');
+  });
+});
+
+describe('handleConversationRelevance with reason generation', () => {
+  it('should generate comprehensive reason when there are irrelevancies', async () => {
+    // Mock provider for verdict generation
+    let callCount = 0;
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockImplementation(async (prompt: string) => {
+        callCount++;
+        // First few calls are verdicts, last one is reason generation
+        if (prompt.includes('irrelevancies')) {
+          // This is the reason generation call
+          return {
+            output: JSON.stringify({
+              reason:
+                'The score is 0.6 because 2 out of 5 responses were irrelevant to the conversation context.',
+            }),
+            tokenUsage: { total: 20, prompt: 10, completion: 10, cached: 0 },
+          };
+        } else {
+          // Verdict calls - with 5 windows and windowSize 3, we evaluate positions 1-5
+          // Make positions 3 and 5 irrelevant (2 out of 5)
+          const isIrrelevant = callCount === 3 || callCount === 5;
+          return {
+            output: JSON.stringify({
+              verdict: isIrrelevant ? 'no' : 'yes',
+              ...(isIrrelevant && {
+                reason: `Response ${callCount} was irrelevant`,
+              }),
+            }),
+            tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+          };
+        }
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const conversation = [
+      { input: 'Question 1', output: 'Answer 1' },
+      { input: 'Question 2', output: 'Answer 2' },
+      { input: 'Question 3', output: 'Answer 3' },
+      { input: 'Question 4', output: 'Answer 4' },
+      { input: 'Question 5', output: 'Answer 5' },
+    ];
+
+    const params: AssertionParams = {
+      baseType: 'conversation-relevance' as const,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+        config: { windowSize: 3 },
+      },
+      assertionValueContext: {
+        vars: {},
+        test: {} as AtomicTestCase,
+        prompt: 'test',
+        logProbs: undefined,
+        provider: mockProvider,
+        providerResponse: { output: 'test' },
+      },
+      output: 'test',
+      outputString: 'test',
+      providerResponse: { output: 'test' },
+      test: {
+        vars: { _conversation: conversation },
+        options: { provider: mockProvider },
+      } as AtomicTestCase,
+      inverse: false,
+    };
+
+    const result = await handleConversationRelevance(params);
+
+    // Should have generated a comprehensive reason
+    expect(result.reason).toContain(
+      'The score is 0.6 because 2 out of 5 responses were irrelevant',
+    );
+    expect(result.tokensUsed).toBeDefined();
+    expect(result.tokensUsed!.total).toBeGreaterThan(0);
+  });
+
+  it('should handle empty conversations gracefully', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ verdict: 'yes' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const params: AssertionParams = {
+      baseType: 'conversation-relevance' as const,
+      assertion: {
+        type: 'conversation-relevance',
+        threshold: 0.8,
+      },
+      assertionValueContext: {
+        vars: {},
+        test: {} as AtomicTestCase,
+        prompt: 'Hello',
+        logProbs: undefined,
+        provider: mockProvider,
+        providerResponse: { output: 'Hi there!' },
+      },
+      output: 'Hi there!',
+      outputString: 'Hi there!',
+      providerResponse: { output: 'Hi there!' },
+      prompt: 'Hello',
+      test: {
+        vars: {},
+        options: { provider: mockProvider },
+      } as AtomicTestCase,
+      inverse: false,
+    };
+
+    const result = await handleConversationRelevance(params);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+});

test/external/matchers.test.ts

@@ -0,0 +1,341 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { matchesConversationRelevance } from '../../src/external/matchers/deepeval';
+import { getDefaultProviders } from '../../src/providers/defaults';
+
+// Mock the text provider
+
+vi.mock('../../src/providers/defaults', () => ({
+  getDefaultProviders: vi.fn(),
+}));
+
+describe('matchesConversationRelevance', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should return pass=true for relevant responses', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          verdict: 'yes',
+        }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'The capital of France is Paris.',
+      },
+    ];
+
+    const result = await matchesConversationRelevance(messages, 0.5);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+    expect(result.tokensUsed).toEqual({
+      total: 10,
+      prompt: 5,
+      completion: 5,
+      cached: 0,
+    });
+  });
+
+  it('should return pass=false for irrelevant responses', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          verdict: 'no',
+          reason: 'Response is completely unrelated to the question',
+        }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'The weather is nice today.',
+      },
+    ];
+
+    const result = await matchesConversationRelevance(messages, 0.5);
+    expect(result.pass).toBe(false);
+    expect(result.score).toBe(0);
+    expect(result.reason).toBe('Response is completely unrelated to the question');
+  });
+
+  it('should consider conversation context', async () => {
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'The capital of France is Paris.',
+      },
+      {
+        input: 'What is its population?',
+        output: 'Paris has a population of about 2.2 million people.',
+      },
+    ];
+
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          verdict: 'yes',
+          reason:
+            'Response directly answers the population question with context from previous message about Paris',
+        }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const result = await matchesConversationRelevance(messages, 0.5);
+    expect(result.pass).toBe(true);
+    // Check that both messages appear in the prompt
+    const prompt = mockProvider.callApi.mock.calls[0][0];
+    expect(prompt).toContain('What is the capital of France?');
+    expect(prompt).toContain('What is its population?');
+  });
+
+  it('should handle provider errors', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        error: 'API error',
+        tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'Paris',
+      },
+    ];
+
+    const result = await matchesConversationRelevance(messages, 0.5);
+    expect(result.pass).toBe(false);
+    expect(result.reason).toBe('API error');
+    expect(result.tokensUsed).toMatchObject({
+      total: 5,
+      prompt: 5,
+      completion: 0,
+      cached: 0,
+    });
+  });
+
+  it('should handle malformed provider responses', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: 'not json',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'Paris',
+      },
+    ];
+
+    const result = await matchesConversationRelevance(messages, 0.5);
+    expect(result.pass).toBe(false);
+    expect(result.reason).toMatch(/Error parsing output/);
+  });
+
+  it('should use custom rubric prompt with {{ messages }} (safe templating boundary)', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({
+          verdict: 'yes',
+        }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'Paris',
+      },
+    ];
+    const customRubric = 'Custom rubric template with {{ messages | dump(2) }}';
+
+    await matchesConversationRelevance(messages, 0.5, {}, { rubricPrompt: customRubric });
+    const prompt = mockProvider.callApi.mock.calls[0][0];
+    // Custom rubric prompt (user config) IS rendered as a template — this is the safe path
+    expect(prompt).toBe('Custom rubric template with ' + JSON.stringify(messages, null, 2));
+  });
+
+  // SSTI regression tests — model output must never be rendered as a Nunjucks template.
+  // See GHSA-vxgq-r84v-2r45.
+
+  it('should not render env variable access in model output', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ verdict: 'yes' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      {
+        input: 'Tell me a secret',
+        output: 'The answer is {{env.OPENAI_API_KEY}} and {{env.AWS_SECRET_ACCESS_KEY}}',
+      },
+    ];
+
+    await matchesConversationRelevance(messages, 0.5, {});
+    const prompt = mockProvider.callApi.mock.calls[0][0];
+    expect(prompt).toContain('{{env.OPENAI_API_KEY}}');
+    expect(prompt).toContain('{{env.AWS_SECRET_ACCESS_KEY}}');
+  });
+
+  it('should not render template syntax in _conversation messages', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: JSON.stringify({ verdict: 'yes' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    // Simulates _conversation history where prior model output contains SSTI payloads
+    const messages = [
+      {
+        input: 'What is {{variable}}?',
+        output:
+          "{{range.constructor(\"return require('child_process').execSync('id').toString()\")()}}",
+      },
+      {
+        input: 'Follow up with {{env.SECRET}}',
+        output: '{% for x in range(10) %}{{x}}{% endfor %}',
+      },
+    ];
+
+    await matchesConversationRelevance(messages, 0.5, { variable: 'test', SECRET: 'val' });
+    const prompt = mockProvider.callApi.mock.calls[0][0];
+    // Both input and output must remain literal — no template evaluation
+    expect(prompt).toContain('What is {{variable}}?');
+    expect(prompt).toContain('range.constructor(');
+    expect(prompt).toContain("execSync('id')");
+    expect(prompt).toContain('Follow up with {{env.SECRET}}');
+    expect(prompt).toContain('{% for x in range(10) %}{{x}}{% endfor %}');
+  });
+
+  it('should handle markdown-formatted JSON responses', async () => {
+    const mockProvider = {
+      id: () => 'mock-provider',
+      callApi: vi.fn().mockResolvedValue({
+        output: '```json\n{"verdict": "yes"}\n```',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+      }),
+    };
+
+    vi.mocked(getDefaultProviders).mockResolvedValue({
+      embeddingProvider: mockProvider,
+      gradingJsonProvider: mockProvider,
+      gradingProvider: mockProvider,
+      llmRubricProvider: mockProvider,
+      moderationProvider: mockProvider,
+      suggestionsProvider: mockProvider,
+      synthesizeProvider: mockProvider,
+    });
+
+    const messages = [
+      {
+        input: 'What is the capital of France?',
+        output: 'Paris',
+      },
+    ];
+
+    const result = await matchesConversationRelevance(messages, 0.5);
+    expect(result.pass).toBe(true);
+    expect(result.score).toBe(1);
+  });
+});