promptfoo--promptfoo/test/models/evalUserRated.test.ts

import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { HUMAN_ASSERTION_TYPE } from '../../src/constants';
import { getDb } from '../../src/database/index';
import { runDbMigrations } from '../../src/migrate';
import { queryTestIndicesOptimized } from '../../src/models/evalPerformance';
import { ResultFailureReason } from '../../src/types/index';
import EvalFactory from '../factories/evalFactory';

import type { EvaluateResult, GradingResult } from '../../src/types/index';

/**
 * Helper to create a GradingResult with a human rating assertion.
 */
function createHumanRatedGradingResult(
  pass: boolean,
  score: number,
  reason: string,
): GradingResult {
  return {
    pass,
    score,
    reason,
    namedScores: {},
    tokensUsed: { total: 10, prompt: 5, completion: 5 },
    componentResults: [
      {
        assertion: {
          type: HUMAN_ASSERTION_TYPE,
        },
        pass,
        score,
        reason,
      },
    ],
  };
}

/**
 * Helper to create a regular (non-human-rated) GradingResult.
 */
function createRegularGradingResult(pass: boolean, score: number): GradingResult {
  return {
    pass,
    score,
    reason: pass ? 'Test passed' : 'Test failed',
    namedScores: {},
    tokensUsed: { total: 10, prompt: 5, completion: 5 },
    componentResults: [
      {
        assertion: {
          type: 'equals',
        },
        pass,
        score,
        reason: pass ? 'Values match' : 'Values differ',
      },
    ],
  };
}

describe('User-Rated Filter Feature', () => {
  beforeAll(async () => {
    await runDbMigrations();
  });

  beforeEach(async () => {
    // Clear all tables before each test
    const db = await getDb();
    await db.run('DELETE FROM eval_results');
    await db.run('DELETE FROM evals_to_datasets');
    await db.run('DELETE FROM evals_to_prompts');
    await db.run('DELETE FROM evals_to_tags');
    await db.run('DELETE FROM evals');
  });

  describe('Eval.queryTestIndices user-rated filter', () => {
    it('should return only user-rated results when filterMode is user-rated', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add results with various user rating scenarios
      const results = [
        { testIdx: 0, hasHumanRating: true, pass: true },
        { testIdx: 1, hasHumanRating: false, pass: true },
        { testIdx: 2, hasHumanRating: true, pass: false },
        { testIdx: 3, hasHumanRating: false, pass: false },
        { testIdx: 4, hasHumanRating: true, pass: true },
      ];

      for (const result of results) {
        await eval_.addResult({
          description: `test-${result.testIdx}`,
          promptIdx: 0,
          testIdx: result.testIdx,
          testCase: { vars: { test: `value${result.testIdx}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${result.testIdx}` },
          response: { output: `Response ${result.testIdx}` },
          error: null,
          failureReason: result.pass ? ResultFailureReason.NONE : ResultFailureReason.ASSERT,
          success: result.pass,
          score: result.pass ? 1 : 0,
          latencyMs: 100,
          gradingResult: result.hasHumanRating
            ? createHumanRatedGradingResult(result.pass, result.pass ? 1 : 0, 'User rated')
            : createRegularGradingResult(result.pass, result.pass ? 1 : 0),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      // Test user-rated filter
      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      // Should return only test indices 0, 2, and 4 (those with human ratings)
      expect(filteredCount).toBe(3);
      expect(testIndices).toEqual([0, 2, 4]);
    });

    it('should handle empty dataset with user-rated filter', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      expect(filteredCount).toBe(0);
      expect(testIndices).toEqual([]);
    });

    it('should handle dataset with no user-rated results', async () => {
      const eval_ = await EvalFactory.create({
        numResults: 5,
        resultTypes: ['success', 'failure'],
      });

      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      expect(filteredCount).toBe(0);
      expect(testIndices).toEqual([]);
    });

    it('should work with pagination when filtering user-rated', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add 10 user-rated results
      for (let i = 0; i < 10; i++) {
        await eval_.addResult({
          description: `test-${i}`,
          promptIdx: 0,
          testIdx: i,
          testCase: { vars: { test: `value${i}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${i}` },
          response: { output: `Response ${i}` },
          error: null,
          failureReason: ResultFailureReason.NONE,
          success: true,
          score: 1,
          latencyMs: 100,
          gradingResult: createHumanRatedGradingResult(true, 1, `User rated item ${i}`),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      // Test pagination
      const page1 = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
        offset: 0,
        limit: 5,
      });

      const page2 = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
        offset: 5,
        limit: 5,
      });

      expect(page1.filteredCount).toBe(10);
      expect(page1.testIndices).toEqual([0, 1, 2, 3, 4]);

      expect(page2.filteredCount).toBe(10);
      expect(page2.testIndices).toEqual([5, 6, 7, 8, 9]);
    });

    it('should combine user-rated filter with search query', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      const testData = [
        { testIdx: 0, hasHumanRating: true, output: 'Response with needle' },
        { testIdx: 1, hasHumanRating: true, output: 'Different response' },
        { testIdx: 2, hasHumanRating: false, output: 'Contains needle' },
        { testIdx: 3, hasHumanRating: true, output: 'Another needle' },
      ];

      for (const data of testData) {
        await eval_.addResult({
          description: `test-${data.testIdx}`,
          promptIdx: 0,
          testIdx: data.testIdx,
          testCase: { vars: { test: `value${data.testIdx}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${data.testIdx}` },
          response: { output: data.output },
          error: null,
          failureReason: ResultFailureReason.NONE,
          success: true,
          score: 1,
          latencyMs: 100,
          gradingResult: data.hasHumanRating
            ? createHumanRatedGradingResult(true, 1, 'User rated')
            : createRegularGradingResult(true, 1),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      // Search for "needle" with user-rated filter
      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
        searchQuery: 'needle',
      });

      // Should return only user-rated results that also contain "needle"
      expect(filteredCount).toBe(2);
      expect(testIndices).toEqual([0, 3]);
    });

    it('should handle malformed grading results gracefully', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });
      const db = await getDb();

      // Insert a result with null grading_result
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0, NULL
        )
      `);

      // Insert a result with empty componentResults
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test2', '${eval_.id}', 0, 1, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": []}'
        )
      `);

      // Insert a valid user-rated result
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test3', '${eval_.id}', 0, 2, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "human"}, "pass": true, "score": 1, "reason": "Good"}]}'
        )
      `);

      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      // Should only return the valid user-rated result
      expect(filteredCount).toBe(1);
      expect(testIndices).toEqual([2]);
    });

    it('should distinguish human assertions from other assertion types', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });
      const db = await getDb();

      // Result with 'human' assertion type (should match)
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "human"}, "pass": true, "score": 1}]}'
        )
      `);

      // Result with 'equals' assertion type (should NOT match)
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test2', '${eval_.id}', 0, 1, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "equals"}, "pass": true, "score": 1}]}'
        )
      `);

      // Result with 'humanoid' assertion type (should NOT match - avoid false positives)
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test3', '${eval_.id}', 0, 2, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "humanoid"}, "pass": true, "score": 1}]}'
        )
      `);

      // Result with 'is-human' assertion type (should NOT match)
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test4', '${eval_.id}', 0, 3, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "is-human"}, "pass": true, "score": 1}]}'
        )
      `);

      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      // Should only return test index 0 (the one with exactly 'human' assertion type)
      expect(filteredCount).toBe(1);
      expect(testIndices).toEqual([0]);
    });
  });

  describe('evalPerformance.queryTestIndicesOptimized user-rated filter', () => {
    it('should return only user-rated results with optimized query', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add test data
      const results = [
        { testIdx: 0, hasHumanRating: true },
        { testIdx: 1, hasHumanRating: false },
        { testIdx: 2, hasHumanRating: true },
      ];

      for (const result of results) {
        await eval_.addResult({
          description: `test-${result.testIdx}`,
          promptIdx: 0,
          testIdx: result.testIdx,
          testCase: { vars: { test: `value${result.testIdx}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${result.testIdx}` },
          response: { output: `Response ${result.testIdx}` },
          error: null,
          failureReason: ResultFailureReason.NONE,
          success: true,
          score: 1,
          latencyMs: 100,
          gradingResult: result.hasHumanRating
            ? createHumanRatedGradingResult(true, 1, 'User rated')
            : createRegularGradingResult(true, 1),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
        filterMode: 'user-rated',
      });

      expect(filteredCount).toBe(2);
      expect(testIndices).toEqual([0, 2]);
    });

    it('should handle large datasets efficiently', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add 100 results, every 5th one is user-rated
      for (let i = 0; i < 100; i++) {
        const hasHumanRating = i % 5 === 0;
        await eval_.addResult({
          description: `test-${i}`,
          promptIdx: 0,
          testIdx: i,
          testCase: { vars: { test: `value${i}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${i}` },
          response: { output: `Response ${i}` },
          error: null,
          failureReason: ResultFailureReason.NONE,
          success: true,
          score: 1,
          latencyMs: 100,
          gradingResult: hasHumanRating
            ? createHumanRatedGradingResult(true, 1, `User rated ${i}`)
            : createRegularGradingResult(true, 1),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      const startTime = Date.now();
      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
        filterMode: 'user-rated',
        limit: 10,
      });
      const duration = Date.now() - startTime;

      // Should have 20 user-rated items (0, 5, 10, 15, ...)
      expect(filteredCount).toBe(20);
      expect(testIndices).toEqual([0, 5, 10, 15, 20, 25, 30, 35, 40, 45]);

      // Query should be fast (typically under 100ms for 100 items)
      expect(duration).toBeLessThan(500);
    });
  });

  describe('Integration with getTablePage', () => {
    it('should filter user-rated correctly through getTablePage', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add mixed results
      for (let i = 0; i < 6; i++) {
        const hasHumanRating = i % 2 === 0;
        await eval_.addResult({
          description: `test-${i}`,
          promptIdx: 0,
          testIdx: i,
          testCase: { vars: { test: `value${i}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${i}` },
          response: { output: `Response ${i}` },
          error: null,
          failureReason: ResultFailureReason.NONE,
          success: true,
          score: 1,
          latencyMs: 100,
          gradingResult: hasHumanRating
            ? createHumanRatedGradingResult(true, 1, `User rated ${i}`)
            : createRegularGradingResult(true, 1),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      const result = await eval_.getTablePage({
        filterMode: 'user-rated',
        filters: [],
      });

      // Should have 3 user-rated results (0, 2, 4)
      expect(result.filteredCount).toBe(3);
      expect(result.body.length).toBe(3);

      // Verify the test indices in the body
      const testIndices = result.body.map((row) => row.testIdx);
      expect(testIndices).toEqual([0, 2, 4]);

      // Verify each result has a human assertion in componentResults
      for (const row of result.body) {
        const hasHumanRating = row.outputs.some((output) =>
          output.gradingResult?.componentResults?.some(
            (cr) => cr.assertion?.type === HUMAN_ASSERTION_TYPE,
          ),
        );
        expect(hasHumanRating).toBe(true);
      }
    });
  });

  describe('Edge cases and error handling', () => {
    it('should handle results with null componentResults', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });
      const db = await getDb();

      // Insert result with grading_result but null componentResults
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "reason": "Test", "componentResults": null}'
        )
      `);

      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      expect(filteredCount).toBe(0);
      expect(testIndices).toEqual([]);
    });

    it('should handle results with multiple componentResults including human', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });
      const db = await getDb();

      // Result with multiple componentResults, one of which is human
      await db.run(`
        INSERT INTO eval_results (
          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
          success, score, grading_result
        ) VALUES (
          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
          '{"pass": true, "score": 1, "componentResults": [
            {"assertion": {"type": "equals"}, "pass": true, "score": 1},
            {"assertion": {"type": "human"}, "pass": true, "score": 1},
            {"assertion": {"type": "contains"}, "pass": true, "score": 1}
          ]}'
        )
      `);

      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
        filterMode: 'user-rated',
      });

      expect(filteredCount).toBe(1);
      expect(testIndices).toEqual([0]);
    });

    it('should handle concurrent access to user-rated filter', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add some user-rated results
      for (let i = 0; i < 10; i++) {
        await eval_.addResult({
          description: `test-${i}`,
          promptIdx: 0,
          testIdx: i,
          testCase: { vars: { test: `value${i}` } },
          promptId: 'test-prompt',
          provider: { id: 'test-provider', label: 'test-label' },
          prompt: { raw: 'Test prompt', label: 'Test prompt' },
          vars: { test: `value${i}` },
          response: { output: `Response ${i}` },
          error: null,
          failureReason: ResultFailureReason.NONE,
          success: true,
          score: 1,
          latencyMs: 100,
          gradingResult: createHumanRatedGradingResult(true, 1, `User rated ${i}`),
          namedScores: {},
          cost: 0.007,
          metadata: {},
        } as EvaluateResult);
      }

      // Run multiple queries concurrently
      const promises = Array.from({ length: 5 }, () =>
        (eval_ as any).queryTestIndices({ filterMode: 'user-rated' }),
      );

      const results = await Promise.all(promises);

      // All concurrent queries should return the same results
      const firstResult = results[0];
      for (const result of results) {
        expect(result.filteredCount).toBe(firstResult.filteredCount);
        expect(result.testIndices).toEqual(firstResult.testIndices);
      }
    });

    it('should verify SQL injection safety', async () => {
      const eval_ = await EvalFactory.create({ numResults: 0 });

      // Add a user-rated result
      await eval_.addResult({
        description: 'test-safe',
        promptIdx: 0,
        testIdx: 0,
        testCase: { vars: { test: 'value' } },
        promptId: 'test-prompt',
        provider: { id: 'test-provider', label: 'test-label' },
        prompt: { raw: 'Test prompt', label: 'Test prompt' },
        vars: { test: 'value' },
        response: { output: 'Safe response' },
        error: null,
        failureReason: ResultFailureReason.NONE,
        success: true,
        score: 1,
        latencyMs: 100,
        gradingResult: createHumanRatedGradingResult(true, 1, 'User rated'),
        namedScores: {},
        cost: 0.007,
        metadata: {},
      } as EvaluateResult);

      // Try SQL injection in search query
      const maliciousSearch = "'; DROP TABLE eval_results; --";
      const { filteredCount } = await queryTestIndicesOptimized(eval_.id, {
        filterMode: 'user-rated',
        searchQuery: maliciousSearch,
      });

      // Should not match the malicious search string
      expect(filteredCount).toBe(0);

      // Test that user-rated filter still works after attempted injection
      const justUserRated = await queryTestIndicesOptimized(eval_.id, {
        filterMode: 'user-rated',
      });
      expect(justUserRated.filteredCount).toBe(1);
      expect(justUserRated.testIndices).toEqual([0]);
    });
  });
});