chore: import upstream snapshot with attribution

test/models/eval.filteredMetrics.test.ts

@@ -0,0 +1,499 @@
+/**
+ * CRITICAL TEST: WHERE clause consistency between pagination and metrics.
+ *
+ * This test ensures that queryTestIndices() and getFilteredMetrics() use
+ * the EXACT SAME WHERE clause logic, preventing silent data corruption where
+ * metrics don't match the displayed results.
+ *
+ * If these tests fail, it means the two methods have diverged and metrics
+ * will be inaccurate!
+ */
+
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { runDbMigrations } from '../../src/migrate';
+import EvalFactory from '../factories/evalFactory';
+
+describe('Filtered Metrics - WHERE Clause Consistency', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(async () => {
+    // Clear all tables before each test
+    const db = await getDb();
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  /**
+   * CRITICAL TEST #1: Row count consistency
+   *
+   * The number of rows in pagination MUST match the sum of test counts in metrics.
+   * If this fails, the WHERE clauses have diverged!
+   */
+  describe('CRITICAL: Row count consistency', () => {
+    it('should return same row count for pagination and metrics with no filters', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({});
+      const metrics = await eval_.getFilteredMetrics({});
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+    });
+
+    it('should return same row count for pagination and metrics with filterMode=errors', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filterMode: 'errors' });
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'errors' });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBeGreaterThan(0); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with filterMode=failures', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filterMode: 'failures' });
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'failures' });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBeGreaterThan(0); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with filterMode=passes', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filterMode: 'passes' });
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'passes' });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBeGreaterThan(0); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with filterMode=highlights', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+        withHighlights: true,
+      });
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'highlights' });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBeGreaterThan(0); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with searchQuery', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+        searchableContent: 'searchable_content',
+      });
+
+      const searchQuery = 'searchable_content';
+      const { testIndices } = await (eval_ as any).queryTestIndices({ searchQuery });
+      const metrics = await eval_.getFilteredMetrics({ searchQuery });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBeGreaterThan(0); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with metadata filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+      });
+
+      // Add metadata to some rows
+      const db = await getDb();
+      await db.run(
+        `UPDATE eval_results SET metadata = json('{"source":"unit"}') WHERE eval_id = '${eval_.id}' AND test_idx IN (1, 3, 5)`,
+      );
+
+      const filters = [
+        JSON.stringify({
+          logicOperator: 'and',
+          type: 'metadata',
+          operator: 'equals',
+          field: 'source',
+          value: 'unit',
+        }),
+      ];
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filters });
+      const metrics = await eval_.getFilteredMetrics({ filters });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBe(3); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with plugin filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const db = await getDb();
+      await db.run(
+        `UPDATE eval_results SET metadata = json('{"pluginId":"harmful:harassment"}') WHERE eval_id = '${eval_.id}' AND test_idx IN (2, 4, 6)`,
+      );
+
+      const filters = [
+        JSON.stringify({
+          logicOperator: 'and',
+          type: 'plugin',
+          operator: 'equals',
+          value: 'harmful',
+        }),
+      ];
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filters });
+      const metrics = await eval_.getFilteredMetrics({ filters });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBe(3); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with strategy filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const db = await getDb();
+      await db.run(
+        `UPDATE eval_results SET metadata = json('{"strategyId":"jailbreak"}') WHERE eval_id = '${eval_.id}' AND test_idx IN (1, 2, 3)`,
+      );
+
+      const filters = [
+        JSON.stringify({
+          logicOperator: 'and',
+          type: 'strategy',
+          operator: 'equals',
+          value: 'jailbreak',
+        }),
+      ];
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filters });
+      const metrics = await eval_.getFilteredMetrics({ filters });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBe(3); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with severity filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const db = await getDb();
+      await db.run(
+        `UPDATE eval_results SET metadata = json('{"severity":"high"}') WHERE eval_id = '${eval_.id}' AND test_idx IN (0, 5)`,
+      );
+
+      const filters = [
+        JSON.stringify({
+          logicOperator: 'and',
+          type: 'severity',
+          operator: 'equals',
+          value: 'high',
+        }),
+      ];
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filters });
+      const metrics = await eval_.getFilteredMetrics({ filters });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBe(2); // Sanity check
+    });
+
+    it('should return same row count for pagination and metrics with metric filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+        withNamedScores: true,
+      });
+
+      const filters = [
+        JSON.stringify({
+          logicOperator: 'and',
+          type: 'metric',
+          operator: 'equals',
+          value: 'accuracy',
+        }),
+      ];
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filters });
+      const metrics = await eval_.getFilteredMetrics({ filters });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+      expect(testIndices.length).toBe(10); // All have named scores
+    });
+
+    it('should return same row count for pagination and metrics with combined filters', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+        withHighlights: true,
+        withNamedScores: true,
+        searchableContent: 'searchable',
+      });
+
+      const db = await getDb();
+      await db.run(
+        `UPDATE eval_results SET metadata = json('{"source":"unit","severity":"high"}') WHERE eval_id = '${eval_.id}' AND test_idx IN (3, 6, 9)`,
+      );
+
+      const filters = [
+        JSON.stringify({
+          logicOperator: 'and',
+          type: 'metadata',
+          operator: 'equals',
+          field: 'source',
+          value: 'unit',
+        }),
+      ];
+
+      const { testIndices } = await (eval_ as any).queryTestIndices({
+        filterMode: 'failures',
+        searchQuery: 'searchable',
+        filters,
+      });
+
+      const metrics = await eval_.getFilteredMetrics({
+        filterMode: 'failures',
+        searchQuery: 'searchable',
+        filters,
+      });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(metricsTotal);
+    });
+  });
+
+  /**
+   * CRITICAL TEST #2: Empty result handling
+   *
+   * When no results match the filter, both methods must return zero counts.
+   */
+  describe('CRITICAL: Empty result handling', () => {
+    it('should return zero counts when no results match filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success'],
+      });
+
+      // Filter for errors when there are none
+      const { testIndices } = await (eval_ as any).queryTestIndices({ filterMode: 'errors' });
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'errors' });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(0);
+      expect(metricsTotal).toBe(0);
+    });
+
+    it('should return zero counts when search matches nothing', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const searchQuery = 'nonexistent_search_term_xyz';
+      const { testIndices } = await (eval_ as any).queryTestIndices({ searchQuery });
+      const metrics = await eval_.getFilteredMetrics({ searchQuery });
+
+      const metricsTotal = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      expect(testIndices.length).toBe(0);
+      expect(metricsTotal).toBe(0);
+    });
+  });
+
+  /**
+   * CRITICAL TEST #3: Metrics correctness
+   *
+   * Not only should the counts match, but the actual metrics should be correct
+   * based on the filtered dataset.
+   */
+  describe('CRITICAL: Metrics correctness', () => {
+    it('should calculate correct metrics for filtered failures', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'], // Cycles: success, error, failure, success, error, failure...
+      });
+
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'failures' });
+
+      // With cycling pattern, we expect roughly 1/3 failures
+      const totalTests = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      const failCount = metrics.reduce((sum, m) => sum + m.testFailCount, 0);
+
+      expect(totalTests).toBeGreaterThan(0);
+      expect(failCount).toBe(totalTests); // All filtered results should be failures
+      expect(metrics[0].testPassCount).toBe(0); // No passes in failure filter
+      expect(metrics[0].testErrorCount).toBe(0); // No errors in failure filter
+    });
+
+    it('should calculate correct metrics for filtered passes', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'passes' });
+
+      const totalTests = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      const passCount = metrics.reduce((sum, m) => sum + m.testPassCount, 0);
+
+      expect(totalTests).toBeGreaterThan(0);
+      expect(passCount).toBe(totalTests); // All filtered results should be passes
+      expect(metrics[0].testFailCount).toBe(0); // No failures in pass filter
+      expect(metrics[0].testErrorCount).toBe(0); // No errors in pass filter
+    });
+
+    it('should calculate correct metrics for filtered errors', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'error', 'failure'],
+      });
+
+      const metrics = await eval_.getFilteredMetrics({ filterMode: 'errors' });
+
+      const totalTests = metrics.reduce(
+        (sum, m) => sum + m.testPassCount + m.testFailCount + m.testErrorCount,
+        0,
+      );
+
+      const errorCount = metrics.reduce((sum, m) => sum + m.testErrorCount, 0);
+
+      expect(totalTests).toBeGreaterThan(0);
+      expect(errorCount).toBe(totalTests); // All filtered results should be errors
+      expect(metrics[0].testPassCount).toBe(0); // No passes in error filter
+      expect(metrics[0].testFailCount).toBe(0); // No failures in error filter
+    });
+
+    it('should include named scores only from filtered results', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+        withNamedScores: true,
+      });
+
+      // All results have accuracy and relevance scores
+      const allMetrics = await eval_.getFilteredMetrics({});
+      const passMetrics = await eval_.getFilteredMetrics({ filterMode: 'passes' });
+
+      // Both should have named scores
+      expect(allMetrics[0].namedScores).toHaveProperty('accuracy');
+      expect(allMetrics[0].namedScores).toHaveProperty('relevance');
+      expect(passMetrics[0].namedScores).toHaveProperty('accuracy');
+      expect(passMetrics[0].namedScores).toHaveProperty('relevance');
+
+      // Pass metrics should have LOWER accuracy sum than all metrics
+      // (5 passes at 1.0 each = 5.0 total) < (5 passes at 1.0 + 5 failures at 0.2 = 6.0 total)
+      expect(passMetrics[0].namedScores.accuracy).toBeLessThan(allMetrics[0].namedScores.accuracy);
+
+      // But the pass metrics should have non-zero accuracy
+      expect(passMetrics[0].namedScores.accuracy).toBeGreaterThan(0);
+    });
+  });
+});

test/models/evalHighlights.test.ts

@@ -0,0 +1,794 @@
+import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { runDbMigrations } from '../../src/migrate';
+import { queryTestIndicesOptimized } from '../../src/models/evalPerformance';
+import { ResultFailureReason } from '../../src/types/index';
+import EvalFactory from '../factories/evalFactory';
+
+import type { EvaluateResult } from '../../src/types/index';
+
+describe('Highlights Filter Feature', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(async () => {
+    // Clear all tables before each test
+    const db = await getDb();
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+  });
+
+  describe('Eval.queryTestIndices highlights filter', () => {
+    it('should return only highlighted results when filterMode is highlights', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add results with various highlighting scenarios
+      const results = [
+        { testIdx: 0, comment: '!highlight This is important' },
+        { testIdx: 1, comment: 'Regular comment' },
+        { testIdx: 2, comment: '!highlight Another highlight' },
+        { testIdx: 3, comment: '' },
+        { testIdx: 4, comment: '!highlight' },
+        { testIdx: 5, comment: 'Not a highlight' },
+      ];
+
+      for (const result of results) {
+        await eval_.addResult({
+          description: `test-${result.testIdx}`,
+          promptIdx: 0,
+          testIdx: result.testIdx,
+          testCase: { vars: { test: `value${result.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${result.testIdx}` },
+          response: { output: `Response ${result.testIdx}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: result.comment,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Test highlights filter
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+
+      // Should return only test indices 0, 2, and 4 (those with !highlight)
+      expect(filteredCount).toBe(3);
+      expect(testIndices).toEqual([0, 2, 4]);
+    });
+
+    it('should handle empty dataset with highlights filter', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should handle dataset with no highlights', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 5,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should work with pagination when filtering highlights', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add 10 highlighted results
+      for (let i = 0; i < 10; i++) {
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: `!highlight Important item ${i}`,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Test pagination
+      const page1 = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+        offset: 0,
+        limit: 5,
+      });
+
+      const page2 = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+        offset: 5,
+        limit: 5,
+      });
+
+      expect(page1.filteredCount).toBe(10);
+      expect(page1.testIndices).toEqual([0, 1, 2, 3, 4]);
+
+      expect(page2.filteredCount).toBe(10);
+      expect(page2.testIndices).toEqual([5, 6, 7, 8, 9]);
+    });
+
+    it('should combine highlights filter with search query', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      const testData = [
+        { testIdx: 0, comment: '!highlight Contains needle', output: 'Response with needle' },
+        { testIdx: 1, comment: '!highlight No match', output: 'Different response' },
+        { testIdx: 2, comment: 'Regular comment', output: 'Contains needle' },
+        { testIdx: 3, comment: '!highlight Has needle too', output: 'Another needle' },
+      ];
+
+      for (const data of testData) {
+        await eval_.addResult({
+          description: `test-${data.testIdx}`,
+          promptIdx: 0,
+          testIdx: data.testIdx,
+          testCase: { vars: { test: `value${data.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${data.testIdx}` },
+          response: { output: data.output },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: data.comment,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Search for "needle" with highlights filter
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+        searchQuery: 'needle',
+      });
+
+      // Should return only highlighted results that also contain "needle"
+      expect(filteredCount).toBe(2);
+      expect(testIndices).toEqual([0, 3]);
+    });
+
+    it('should handle malformed grading results gracefully', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+      const db = await getDb();
+
+      // Insert a result with null grading_result
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0, NULL
+        )
+      `);
+
+      // Insert a result with invalid JSON in grading_result
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test2', '${eval_.id}', 0, 1, '{}', '{}', '{}', 1, 1.0, '{}'
+        )
+      `);
+
+      // Insert a valid highlighted result
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test3', '${eval_.id}', 0, 2, '{}', '{}', '{}', 1, 1.0,
+          '{"comment": "!highlight Valid highlight", "pass": true, "score": 1}'
+        )
+      `);
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+
+      // Should only return the valid highlighted result
+      expect(filteredCount).toBe(1);
+      expect(testIndices).toEqual([2]);
+    });
+
+    it('should combine highlights filter with other filter modes correctly', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Create mixed results
+      const testData = [
+        { testIdx: 0, success: true, comment: '!highlight Success highlight' },
+        { testIdx: 1, success: false, comment: '!highlight Failure highlight' },
+        { testIdx: 2, success: true, comment: 'Regular success' },
+        { testIdx: 3, success: false, comment: 'Regular failure' },
+      ];
+
+      for (const data of testData) {
+        await eval_.addResult({
+          description: `test-${data.testIdx}`,
+          promptIdx: 0,
+          testIdx: data.testIdx,
+          testCase: { vars: { test: `value${data.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${data.testIdx}` },
+          response: { output: `Response ${data.testIdx}` },
+          error: null,
+          failureReason: data.success ? ResultFailureReason.NONE : ResultFailureReason.ASSERT,
+          success: data.success,
+          score: data.success ? 1 : 0,
+          latencyMs: 100,
+          gradingResult: {
+            pass: data.success,
+            score: data.success ? 1 : 0,
+            reason: data.success ? 'Test passed' : 'Test failed',
+            comment: data.comment,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Test highlights filter alone
+      const highlights = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+      expect(highlights.testIndices).toEqual([0, 1]);
+
+      // Verify other filters still work
+      const passes = await (eval_ as any).queryTestIndices({
+        filterMode: 'passes',
+      });
+      expect(passes.testIndices).toEqual([0, 2]);
+
+      const failures = await (eval_ as any).queryTestIndices({
+        filterMode: 'failures',
+      });
+      expect(failures.testIndices).toEqual([1, 3]);
+    });
+  });
+
+  describe('evalPerformance.queryTestIndicesOptimized highlights filter', () => {
+    it('should return only highlighted results with optimized query', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add test data
+      const results = [
+        { testIdx: 0, comment: '!highlight Performance test' },
+        { testIdx: 1, comment: 'Not highlighted' },
+        { testIdx: 2, comment: '!highlight Another highlight' },
+      ];
+
+      for (const result of results) {
+        await eval_.addResult({
+          description: `test-${result.testIdx}`,
+          promptIdx: 0,
+          testIdx: result.testIdx,
+          testCase: { vars: { test: `value${result.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${result.testIdx}` },
+          response: { output: `Response ${result.testIdx}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: result.comment,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+      });
+
+      expect(filteredCount).toBe(2);
+      expect(testIndices).toEqual([0, 2]);
+    });
+
+    it('should handle large datasets efficiently', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add 100 results, every 5th one is highlighted
+      for (let i = 0; i < 100; i++) {
+        const isHighlighted = i % 5 === 0;
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: isHighlighted ? `!highlight Item ${i}` : `Regular item ${i}`,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const startTime = Date.now();
+      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        limit: 10,
+      });
+      const duration = Date.now() - startTime;
+
+      // Should have 20 highlighted items (0, 5, 10, 15, ...)
+      expect(filteredCount).toBe(20);
+      expect(testIndices).toEqual([0, 5, 10, 15, 20, 25, 30, 35, 40, 45]);
+
+      // Query should be fast (typically under 100ms for 100 items)
+      expect(duration).toBeLessThan(500);
+    });
+
+    it('should work with search query in optimized mode', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      const testData = [
+        { testIdx: 0, comment: '!highlight Contains target', output: 'Response with target' },
+        { testIdx: 1, comment: '!highlight No match', output: 'Different response' },
+        { testIdx: 2, comment: 'Regular comment', output: 'Contains target' },
+        { testIdx: 3, comment: '!highlight Also has target', output: 'Another target' },
+      ];
+
+      for (const data of testData) {
+        await eval_.addResult({
+          description: `test-${data.testIdx}`,
+          promptIdx: 0,
+          testIdx: data.testIdx,
+          testCase: { vars: { test: `value${data.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${data.testIdx}` },
+          response: { output: data.output },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: data.comment,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Test 1: With highlights filter and no filters array, search should apply
+      const withSearch = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        searchQuery: 'target',
+      });
+
+      // Search in response field for 'target' AND highlights filter
+      // Indices 0 and 3 have both !highlight and 'target' in output
+      expect(withSearch.filteredCount).toBe(2);
+      expect(withSearch.testIndices).toEqual([0, 3]);
+
+      // Test 2: With highlights filter but empty filters array, search should still apply
+      const withEmptyFilters = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        searchQuery: 'target',
+        filters: [],
+      });
+
+      expect(withEmptyFilters.filteredCount).toBe(2);
+      expect(withEmptyFilters.testIndices).toEqual([0, 3]);
+    });
+
+    it('should handle pagination correctly in optimized mode', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add 15 highlighted results
+      for (let i = 0; i < 15; i++) {
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: `!highlight Important ${i}`,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Get three pages
+      const page1 = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        offset: 0,
+        limit: 5,
+      });
+
+      const page2 = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        offset: 5,
+        limit: 5,
+      });
+
+      const page3 = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        offset: 10,
+        limit: 5,
+      });
+
+      expect(page1.filteredCount).toBe(15);
+      expect(page1.testIndices).toEqual([0, 1, 2, 3, 4]);
+
+      expect(page2.filteredCount).toBe(15);
+      expect(page2.testIndices).toEqual([5, 6, 7, 8, 9]);
+
+      expect(page3.filteredCount).toBe(15);
+      expect(page3.testIndices).toEqual([10, 11, 12, 13, 14]);
+    });
+
+    it('should handle empty results gracefully in optimized mode', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 3,
+        resultTypes: ['success', 'failure'],
+      });
+
+      // No highlights in the default factory data
+      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should verify SQL injection safety in optimized mode', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add a highlighted result
+      await eval_.addResult({
+        description: 'test-safe',
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: { test: 'value' } },
+        promptId: 'test-prompt',
+        provider: { id: 'test-provider', label: 'test-label' },
+        prompt: { raw: 'Test prompt', label: 'Test prompt' },
+        vars: { test: 'value' },
+        response: { output: 'Safe response without injection text' },
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 100,
+        gradingResult: {
+          pass: true,
+          score: 1,
+          reason: 'Test passed',
+          comment: '!highlight Safe result',
+          namedScores: {},
+          tokensUsed: { total: 10, prompt: 5, completion: 5 },
+          componentResults: [],
+        },
+        namedScores: {},
+        cost: 0.007,
+        metadata: {},
+      } as EvaluateResult);
+
+      // Try SQL injection in search query
+      const maliciousSearch = "'; DROP TABLE eval_results; --";
+      const { filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+        searchQuery: maliciousSearch,
+      });
+
+      // Should still work without errors - the search won't find matches but highlights filter still applies
+      // The SQL injection attempt is safely sanitized and won't match the response text
+      // But the highlight filter will still return the highlighted result
+      expect(filteredCount).toBe(0); // No results match the malicious search string
+
+      // Test that highlights filter alone still works after attempted injection
+      const justHighlights = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'highlights',
+      });
+      expect(justHighlights.filteredCount).toBe(1);
+      expect(justHighlights.testIndices).toEqual([0]);
+    });
+  });
+
+  describe('Integration with getTablePage', () => {
+    it('should filter highlights correctly through getTablePage', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add mixed results
+      for (let i = 0; i < 6; i++) {
+        const isHighlighted = i % 2 === 0;
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: isHighlighted ? `!highlight Important ${i}` : `Regular ${i}`,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const result = await eval_.getTablePage({
+        filterMode: 'highlights',
+        filters: [],
+      });
+
+      // Should have 3 highlighted results (0, 2, 4)
+      expect(result.filteredCount).toBe(3);
+      expect(result.body.length).toBe(3);
+
+      // Verify the test indices in the body
+      const testIndices = result.body.map((row) => row.testIdx);
+      expect(testIndices).toEqual([0, 2, 4]);
+
+      // Verify each result has the highlight comment
+      for (const row of result.body) {
+        const hasHighlight = row.outputs.some((output) =>
+          output.gradingResult?.comment?.startsWith('!highlight'),
+        );
+        expect(hasHighlight).toBe(true);
+      }
+    });
+
+    it('should maintain correct counts with highlights filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 10,
+        resultTypes: ['success', 'failure'],
+        withHighlights: true, // This adds highlights to every 3rd result
+      });
+
+      const allResults = await eval_.getTablePage({ filters: [] });
+      const highlightedResults = await eval_.getTablePage({
+        filterMode: 'highlights',
+        filters: [],
+      });
+
+      // Total count should remain the same
+      expect(highlightedResults.totalCount).toBe(allResults.totalCount);
+
+      // Filtered count should be less than total
+      expect(highlightedResults.filteredCount).toBeLessThan(highlightedResults.totalCount);
+
+      // With factory settings, every 3rd result is highlighted (0, 3, 6, 9)
+      expect(highlightedResults.filteredCount).toBe(4);
+    });
+  });
+
+  describe('Edge cases and error handling', () => {
+    it('should handle results with undefined comment field', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+      const db = await getDb();
+
+      // Insert result with grading_result but no comment field
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "reason": "Test"}'
+        )
+      `);
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should handle special characters in highlight comments', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      const specialComments = [
+        { testIdx: 0, comment: `!highlight With 'quotes'` },
+        { testIdx: 1, comment: `!highlight With "double quotes"` },
+        { testIdx: 2, comment: `!highlight With % percent` },
+        { testIdx: 3, comment: `!highlight With _ underscore` },
+        { testIdx: 4, comment: `!highlight With \\ backslash` },
+      ];
+
+      for (const data of specialComments) {
+        await eval_.addResult({
+          description: `test-${data.testIdx}`,
+          promptIdx: 0,
+          testIdx: data.testIdx,
+          testCase: { vars: { test: `value${data.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${data.testIdx}` },
+          response: { output: `Response ${data.testIdx}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Test passed',
+            comment: data.comment,
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'highlights',
+      });
+
+      // All should be matched despite special characters
+      expect(filteredCount).toBe(5);
+      expect(testIndices).toEqual([0, 1, 2, 3, 4]);
+    });
+
+    it('should handle concurrent access to highlights filter', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 20,
+        resultTypes: ['success', 'failure'],
+        withHighlights: true,
+      });
+
+      // Run multiple queries concurrently
+      const promises = Array.from({ length: 5 }, () =>
+        (eval_ as any).queryTestIndices({ filterMode: 'highlights' }),
+      );
+
+      const results = await Promise.all(promises);
+
+      // All concurrent queries should return the same results
+      const firstResult = results[0];
+      for (const result of results) {
+        expect(result.filteredCount).toBe(firstResult.filteredCount);
+        expect(result.testIndices).toEqual(firstResult.testIndices);
+      }
+    });
+  });
+});

test/models/evalMutation.test.ts

@@ -0,0 +1,67 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { updateSignalFile, updateSignalFileForDeletedEvals } from '../../src/database/signal';
+import {
+  invalidateEvaluationCache,
+  notifyEvaluationChanged,
+  notifyEvaluationsDeleted,
+} from '../../src/models/evalMutation';
+import { clearCountCache } from '../../src/models/evalPerformance';
+import { clearStandaloneEvalCache } from '../../src/util/standaloneEvalCache';
+
+vi.mock('../../src/database/signal', () => ({
+  updateSignalFile: vi.fn(),
+  updateSignalFileForDeletedEvals: vi.fn(),
+}));
+
+vi.mock('../../src/util/standaloneEvalCache', () => ({
+  clearStandaloneEvalCache: vi.fn(),
+}));
+
+vi.mock('../../src/models/evalPerformance', () => ({
+  clearCountCache: vi.fn(),
+}));
+
+describe('eval mutation invalidation', () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('invalidates the standalone evaluation cache', () => {
+    invalidateEvaluationCache();
+
+    expect(clearStandaloneEvalCache).toHaveBeenCalledOnce();
+    expect(clearCountCache).toHaveBeenCalledWith(undefined);
+    expect(updateSignalFile).not.toHaveBeenCalled();
+  });
+
+  it('invalidates cached evaluation lists before notifying watchers', () => {
+    notifyEvaluationChanged('eval-123');
+
+    expect(clearStandaloneEvalCache).toHaveBeenCalledOnce();
+    expect(clearCountCache).toHaveBeenCalledWith('eval-123');
+    expect(updateSignalFile).toHaveBeenCalledWith('eval-123');
+    expect(vi.mocked(clearStandaloneEvalCache).mock.invocationCallOrder[0]).toBeLessThan(
+      vi.mocked(updateSignalFile).mock.invocationCallOrder[0],
+    );
+  });
+
+  it('clears the standalone cache and only the deleted evals counts before notifying watchers', () => {
+    notifyEvaluationsDeleted(['eval-123', 'eval-456']);
+
+    expect(clearStandaloneEvalCache).toHaveBeenCalledOnce();
+    // Scoped to the deleted evals — a single delete must not flush every eval's count cache.
+    expect(clearCountCache).toHaveBeenCalledWith('eval-123');
+    expect(clearCountCache).toHaveBeenCalledWith('eval-456');
+    expect(clearCountCache).not.toHaveBeenCalledWith(undefined);
+    expect(updateSignalFileForDeletedEvals).toHaveBeenCalledWith(['eval-123', 'eval-456']);
+  });
+
+  it('flushes every count cache when all evals are deleted (no ids)', () => {
+    notifyEvaluationsDeleted();
+
+    expect(clearStandaloneEvalCache).toHaveBeenCalledOnce();
+    // A single unscoped clearCountCache() call wipes every eval's counts.
+    expect(clearCountCache).toHaveBeenCalledTimes(1);
+    expect(updateSignalFileForDeletedEvals).toHaveBeenCalledWith(undefined);
+  });
+});

test/models/evalPerformance.test.ts

@@ -0,0 +1,287 @@
+import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { getDb } from '../../src/database/index';
+import { runDbMigrations } from '../../src/migrate';
+import Eval from '../../src/models/eval';
+import {
+  clearCountCache,
+  getCachedResultsCount,
+  getTotalResultRowCount,
+} from '../../src/models/evalPerformance';
+import { ResultFailureReason } from '../../src/types/index';
+
+describe('evalPerformance', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(async () => {
+    const db = await getDb();
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+    clearCountCache();
+  });
+
+  /**
+   * Helper to create an eval and add results for provider x test combinations.
+   * Returns the eval and the expected counts.
+   */
+  async function createEvalWithResults(numProviders: number, numTests: number) {
+    const providers = Array.from({ length: numProviders }, (_, i) => ({ id: `provider-${i + 1}` }));
+    const tests = Array.from({ length: numTests }, (_, i) => ({ vars: { input: `test${i + 1}` } }));
+
+    const eval_ = await Eval.create(
+      {
+        providers,
+        prompts: ['Test prompt'],
+        tests,
+      },
+      [{ raw: 'Test prompt', label: 'Test prompt' }],
+    );
+
+    // Add results for each provider × test combination
+    for (let providerIdx = 0; providerIdx < numProviders; providerIdx++) {
+      for (let testIdx = 0; testIdx < numTests; testIdx++) {
+        await eval_.addResult({
+          description: `test-${providerIdx}-${testIdx}`,
+          promptIdx: 0,
+          testIdx,
+          testCase: { vars: { input: `test${testIdx + 1}` } },
+          promptId: 'test-prompt',
+          provider: { id: `provider-${providerIdx + 1}`, label: `Provider ${providerIdx + 1}` },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { input: `test${testIdx + 1}` },
+          response: {
+            output: `response-${providerIdx}-${testIdx}`,
+            tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+          },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: {
+            pass: true,
+            score: 1,
+            reason: 'Pass',
+            namedScores: {},
+            tokensUsed: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            componentResults: [],
+          },
+          namedScores: {},
+          cost: 0.001,
+          metadata: {},
+        });
+      }
+    }
+
+    return {
+      eval_,
+      expectedDistinctCount: numTests,
+      expectedTotalRowCount: numProviders * numTests,
+    };
+  }
+
+  describe('getCachedResultsCount', () => {
+    it('should count distinct test indices (unique test cases)', async () => {
+      // Create an eval with 2 providers and 3 test cases
+      // This should produce 6 total results (2 providers × 3 tests)
+      // But only 3 distinct test indices
+      const { eval_, expectedDistinctCount } = await createEvalWithResults(2, 3);
+
+      // Should return 3 (distinct test indices), not 6 (total rows)
+      const count = await getCachedResultsCount(eval_.id);
+      expect(count).toBe(expectedDistinctCount);
+    });
+
+    it('should return 0 for an eval with no results', async () => {
+      const eval_ = await Eval.create(
+        {
+          providers: [{ id: 'provider-1' }],
+          prompts: ['Test prompt'],
+          tests: [{ vars: { input: 'test1' } }],
+        },
+        [{ raw: 'Test prompt', label: 'Test prompt' }],
+      );
+
+      const count = await getCachedResultsCount(eval_.id);
+      expect(count).toBe(0);
+    });
+
+    it('should cache the count result', async () => {
+      const { eval_ } = await createEvalWithResults(1, 1);
+
+      // First call should hit the database
+      const count1 = await getCachedResultsCount(eval_.id);
+      expect(count1).toBe(1);
+
+      // Second call should return cached result
+      const count2 = await getCachedResultsCount(eval_.id);
+      expect(count2).toBe(1);
+
+      // Clear cache and verify we can get fresh count
+      clearCountCache(eval_.id);
+      const count3 = await getCachedResultsCount(eval_.id);
+      expect(count3).toBe(1);
+    });
+  });
+
+  describe('getTotalResultRowCount', () => {
+    it('should count all result rows (including multiple per test)', async () => {
+      // Create an eval with 2 providers and 3 test cases
+      // This should produce 6 total result rows (2 providers × 3 tests)
+      const { eval_, expectedTotalRowCount } = await createEvalWithResults(2, 3);
+
+      // Should return 6 (total rows), not 3 (distinct test indices)
+      const count = await getTotalResultRowCount(eval_.id);
+      expect(count).toBe(expectedTotalRowCount);
+    });
+
+    it('should return 0 for an eval with no results', async () => {
+      const eval_ = await Eval.create(
+        {
+          providers: [{ id: 'provider-1' }],
+          prompts: ['Test prompt'],
+          tests: [{ vars: { input: 'test1' } }],
+        },
+        [{ raw: 'Test prompt', label: 'Test prompt' }],
+      );
+
+      const count = await getTotalResultRowCount(eval_.id);
+      expect(count).toBe(0);
+    });
+
+    it('should cache the count result', async () => {
+      const { eval_ } = await createEvalWithResults(1, 1);
+
+      // First call should hit the database
+      const count1 = await getTotalResultRowCount(eval_.id);
+      expect(count1).toBe(1);
+
+      // Second call should return cached result
+      const count2 = await getTotalResultRowCount(eval_.id);
+      expect(count2).toBe(1);
+
+      // Clear cache and verify we can get fresh count
+      clearCountCache(eval_.id);
+      const count3 = await getTotalResultRowCount(eval_.id);
+      expect(count3).toBe(1);
+    });
+  });
+
+  describe('count functions comparison', () => {
+    it('should return same count when 1 provider per test', async () => {
+      const { eval_ } = await createEvalWithResults(1, 5);
+
+      const distinctCount = await getCachedResultsCount(eval_.id);
+      const totalCount = await getTotalResultRowCount(eval_.id);
+
+      // With 1 provider, distinct count equals total count
+      expect(distinctCount).toBe(5);
+      expect(totalCount).toBe(5);
+    });
+
+    it('should return different counts when multiple providers per test', async () => {
+      const { eval_ } = await createEvalWithResults(3, 4);
+
+      const distinctCount = await getCachedResultsCount(eval_.id);
+      const totalCount = await getTotalResultRowCount(eval_.id);
+
+      // With 3 providers and 4 tests:
+      // - distinct count = 4 (unique test indices)
+      // - total count = 12 (3 providers × 4 tests)
+      expect(distinctCount).toBe(4);
+      expect(totalCount).toBe(12);
+    });
+  });
+
+  describe('cache invalidation on result write (issue #9348)', () => {
+    const makeResult = (_evalId: string, testIdx: number) =>
+      ({
+        description: `test-${testIdx}`,
+        promptIdx: 0,
+        testIdx,
+        testCase: { vars: { input: `test${testIdx}` } },
+        promptId: 'test-prompt',
+        provider: { id: 'provider-1', label: 'Provider 1' },
+        prompt: { raw: 'Test prompt', label: 'Test prompt' },
+        vars: { input: `test${testIdx}` },
+        response: {
+          output: `response-${testIdx}`,
+          tokenUsage: { total: 0, prompt: 0, completion: 0, cached: 0 },
+        },
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 10,
+        gradingResult: {
+          pass: true,
+          score: 1,
+          reason: 'Pass',
+          namedScores: {},
+          tokensUsed: { total: 0, prompt: 0, completion: 0, cached: 0 },
+          componentResults: [],
+        },
+        namedScores: {},
+        cost: 0,
+        metadata: {},
+      }) as Parameters<
+        typeof import('../../src/models/evalResult')['default']['createFromEvaluateResult']
+      >[1];
+
+    it('getCachedResultsCount refreshes after createFromEvaluateResult (issue #9348)', async () => {
+      const eval_ = await Eval.create(
+        { providers: [{ id: 'provider-1' }], prompts: ['Test prompt'], tests: [] },
+        [{ raw: 'Test prompt', label: 'Test prompt' }],
+      );
+
+      // Seed zero into cache before any inserts
+      const before = await getCachedResultsCount(eval_.id);
+      expect(before).toBe(0);
+
+      // Insert a result — this should bust the cache automatically
+      await eval_.addResult(makeResult(eval_.id, 0));
+
+      // Must see fresh count without any manual clearCountCache() call
+      const after = await getCachedResultsCount(eval_.id);
+      expect(after).toBe(1);
+    });
+
+    it('getTotalResultRowCount refreshes after createFromEvaluateResult (issue #9348)', async () => {
+      const eval_ = await Eval.create(
+        { providers: [{ id: 'provider-1' }], prompts: ['Test prompt'], tests: [] },
+        [{ raw: 'Test prompt', label: 'Test prompt' }],
+      );
+
+      const before = await getTotalResultRowCount(eval_.id);
+      expect(before).toBe(0);
+
+      await eval_.addResult(makeResult(eval_.id, 0));
+
+      const after = await getTotalResultRowCount(eval_.id);
+      expect(after).toBe(1);
+    });
+
+    it('getCachedResultsCount refreshes after createManyFromEvaluateResult (issue #9348)', async () => {
+      const eval_ = await Eval.create(
+        { providers: [{ id: 'provider-1' }], prompts: ['Test prompt'], tests: [] },
+        [{ raw: 'Test prompt', label: 'Test prompt' }],
+      );
+
+      const before = await getCachedResultsCount(eval_.id);
+      expect(before).toBe(0);
+
+      const EvalResult = (await import('../../src/models/evalResult')).default;
+      await EvalResult.createManyFromEvaluateResult(
+        [makeResult(eval_.id, 0), makeResult(eval_.id, 1)] as any,
+        eval_.id,
+      );
+
+      const after = await getCachedResultsCount(eval_.id);
+      expect(after).toBe(2);
+    });
+  });
+});

test/models/evalUserRated.test.ts

@@ -0,0 +1,627 @@
+import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { HUMAN_ASSERTION_TYPE } from '../../src/constants';
+import { getDb } from '../../src/database/index';
+import { runDbMigrations } from '../../src/migrate';
+import { queryTestIndicesOptimized } from '../../src/models/evalPerformance';
+import { ResultFailureReason } from '../../src/types/index';
+import EvalFactory from '../factories/evalFactory';
+
+import type { EvaluateResult, GradingResult } from '../../src/types/index';
+
+/**
+ * Helper to create a GradingResult with a human rating assertion.
+ */
+function createHumanRatedGradingResult(
+  pass: boolean,
+  score: number,
+  reason: string,
+): GradingResult {
+  return {
+    pass,
+    score,
+    reason,
+    namedScores: {},
+    tokensUsed: { total: 10, prompt: 5, completion: 5 },
+    componentResults: [
+      {
+        assertion: {
+          type: HUMAN_ASSERTION_TYPE,
+        },
+        pass,
+        score,
+        reason,
+      },
+    ],
+  };
+}
+
+/**
+ * Helper to create a regular (non-human-rated) GradingResult.
+ */
+function createRegularGradingResult(pass: boolean, score: number): GradingResult {
+  return {
+    pass,
+    score,
+    reason: pass ? 'Test passed' : 'Test failed',
+    namedScores: {},
+    tokensUsed: { total: 10, prompt: 5, completion: 5 },
+    componentResults: [
+      {
+        assertion: {
+          type: 'equals',
+        },
+        pass,
+        score,
+        reason: pass ? 'Values match' : 'Values differ',
+      },
+    ],
+  };
+}
+
+describe('User-Rated Filter Feature', () => {
+  beforeAll(async () => {
+    await runDbMigrations();
+  });
+
+  beforeEach(async () => {
+    // Clear all tables before each test
+    const db = await getDb();
+    await db.run('DELETE FROM eval_results');
+    await db.run('DELETE FROM evals_to_datasets');
+    await db.run('DELETE FROM evals_to_prompts');
+    await db.run('DELETE FROM evals_to_tags');
+    await db.run('DELETE FROM evals');
+  });
+
+  describe('Eval.queryTestIndices user-rated filter', () => {
+    it('should return only user-rated results when filterMode is user-rated', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add results with various user rating scenarios
+      const results = [
+        { testIdx: 0, hasHumanRating: true, pass: true },
+        { testIdx: 1, hasHumanRating: false, pass: true },
+        { testIdx: 2, hasHumanRating: true, pass: false },
+        { testIdx: 3, hasHumanRating: false, pass: false },
+        { testIdx: 4, hasHumanRating: true, pass: true },
+      ];
+
+      for (const result of results) {
+        await eval_.addResult({
+          description: `test-${result.testIdx}`,
+          promptIdx: 0,
+          testIdx: result.testIdx,
+          testCase: { vars: { test: `value${result.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${result.testIdx}` },
+          response: { output: `Response ${result.testIdx}` },
+          error: null,
+          failureReason: result.pass ? ResultFailureReason.NONE : ResultFailureReason.ASSERT,
+          success: result.pass,
+          score: result.pass ? 1 : 0,
+          latencyMs: 100,
+          gradingResult: result.hasHumanRating
+            ? createHumanRatedGradingResult(result.pass, result.pass ? 1 : 0, 'User rated')
+            : createRegularGradingResult(result.pass, result.pass ? 1 : 0),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Test user-rated filter
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      // Should return only test indices 0, 2, and 4 (those with human ratings)
+      expect(filteredCount).toBe(3);
+      expect(testIndices).toEqual([0, 2, 4]);
+    });
+
+    it('should handle empty dataset with user-rated filter', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should handle dataset with no user-rated results', async () => {
+      const eval_ = await EvalFactory.create({
+        numResults: 5,
+        resultTypes: ['success', 'failure'],
+      });
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should work with pagination when filtering user-rated', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add 10 user-rated results
+      for (let i = 0; i < 10; i++) {
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: createHumanRatedGradingResult(true, 1, `User rated item ${i}`),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Test pagination
+      const page1 = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+        offset: 0,
+        limit: 5,
+      });
+
+      const page2 = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+        offset: 5,
+        limit: 5,
+      });
+
+      expect(page1.filteredCount).toBe(10);
+      expect(page1.testIndices).toEqual([0, 1, 2, 3, 4]);
+
+      expect(page2.filteredCount).toBe(10);
+      expect(page2.testIndices).toEqual([5, 6, 7, 8, 9]);
+    });
+
+    it('should combine user-rated filter with search query', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      const testData = [
+        { testIdx: 0, hasHumanRating: true, output: 'Response with needle' },
+        { testIdx: 1, hasHumanRating: true, output: 'Different response' },
+        { testIdx: 2, hasHumanRating: false, output: 'Contains needle' },
+        { testIdx: 3, hasHumanRating: true, output: 'Another needle' },
+      ];
+
+      for (const data of testData) {
+        await eval_.addResult({
+          description: `test-${data.testIdx}`,
+          promptIdx: 0,
+          testIdx: data.testIdx,
+          testCase: { vars: { test: `value${data.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${data.testIdx}` },
+          response: { output: data.output },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: data.hasHumanRating
+            ? createHumanRatedGradingResult(true, 1, 'User rated')
+            : createRegularGradingResult(true, 1),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Search for "needle" with user-rated filter
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+        searchQuery: 'needle',
+      });
+
+      // Should return only user-rated results that also contain "needle"
+      expect(filteredCount).toBe(2);
+      expect(testIndices).toEqual([0, 3]);
+    });
+
+    it('should handle malformed grading results gracefully', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+      const db = await getDb();
+
+      // Insert a result with null grading_result
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0, NULL
+        )
+      `);
+
+      // Insert a result with empty componentResults
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test2', '${eval_.id}', 0, 1, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": []}'
+        )
+      `);
+
+      // Insert a valid user-rated result
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test3', '${eval_.id}', 0, 2, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "human"}, "pass": true, "score": 1, "reason": "Good"}]}'
+        )
+      `);
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      // Should only return the valid user-rated result
+      expect(filteredCount).toBe(1);
+      expect(testIndices).toEqual([2]);
+    });
+
+    it('should distinguish human assertions from other assertion types', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+      const db = await getDb();
+
+      // Result with 'human' assertion type (should match)
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "human"}, "pass": true, "score": 1}]}'
+        )
+      `);
+
+      // Result with 'equals' assertion type (should NOT match)
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test2', '${eval_.id}', 0, 1, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "equals"}, "pass": true, "score": 1}]}'
+        )
+      `);
+
+      // Result with 'humanoid' assertion type (should NOT match - avoid false positives)
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test3', '${eval_.id}', 0, 2, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "humanoid"}, "pass": true, "score": 1}]}'
+        )
+      `);
+
+      // Result with 'is-human' assertion type (should NOT match)
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test4', '${eval_.id}', 0, 3, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": [{"assertion": {"type": "is-human"}, "pass": true, "score": 1}]}'
+        )
+      `);
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      // Should only return test index 0 (the one with exactly 'human' assertion type)
+      expect(filteredCount).toBe(1);
+      expect(testIndices).toEqual([0]);
+    });
+  });
+
+  describe('evalPerformance.queryTestIndicesOptimized user-rated filter', () => {
+    it('should return only user-rated results with optimized query', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add test data
+      const results = [
+        { testIdx: 0, hasHumanRating: true },
+        { testIdx: 1, hasHumanRating: false },
+        { testIdx: 2, hasHumanRating: true },
+      ];
+
+      for (const result of results) {
+        await eval_.addResult({
+          description: `test-${result.testIdx}`,
+          promptIdx: 0,
+          testIdx: result.testIdx,
+          testCase: { vars: { test: `value${result.testIdx}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${result.testIdx}` },
+          response: { output: `Response ${result.testIdx}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: result.hasHumanRating
+            ? createHumanRatedGradingResult(true, 1, 'User rated')
+            : createRegularGradingResult(true, 1),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'user-rated',
+      });
+
+      expect(filteredCount).toBe(2);
+      expect(testIndices).toEqual([0, 2]);
+    });
+
+    it('should handle large datasets efficiently', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add 100 results, every 5th one is user-rated
+      for (let i = 0; i < 100; i++) {
+        const hasHumanRating = i % 5 === 0;
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: hasHumanRating
+            ? createHumanRatedGradingResult(true, 1, `User rated ${i}`)
+            : createRegularGradingResult(true, 1),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const startTime = Date.now();
+      const { testIndices, filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'user-rated',
+        limit: 10,
+      });
+      const duration = Date.now() - startTime;
+
+      // Should have 20 user-rated items (0, 5, 10, 15, ...)
+      expect(filteredCount).toBe(20);
+      expect(testIndices).toEqual([0, 5, 10, 15, 20, 25, 30, 35, 40, 45]);
+
+      // Query should be fast (typically under 100ms for 100 items)
+      expect(duration).toBeLessThan(500);
+    });
+  });
+
+  describe('Integration with getTablePage', () => {
+    it('should filter user-rated correctly through getTablePage', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add mixed results
+      for (let i = 0; i < 6; i++) {
+        const hasHumanRating = i % 2 === 0;
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: hasHumanRating
+            ? createHumanRatedGradingResult(true, 1, `User rated ${i}`)
+            : createRegularGradingResult(true, 1),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      const result = await eval_.getTablePage({
+        filterMode: 'user-rated',
+        filters: [],
+      });
+
+      // Should have 3 user-rated results (0, 2, 4)
+      expect(result.filteredCount).toBe(3);
+      expect(result.body.length).toBe(3);
+
+      // Verify the test indices in the body
+      const testIndices = result.body.map((row) => row.testIdx);
+      expect(testIndices).toEqual([0, 2, 4]);
+
+      // Verify each result has a human assertion in componentResults
+      for (const row of result.body) {
+        const hasHumanRating = row.outputs.some((output) =>
+          output.gradingResult?.componentResults?.some(
+            (cr) => cr.assertion?.type === HUMAN_ASSERTION_TYPE,
+          ),
+        );
+        expect(hasHumanRating).toBe(true);
+      }
+    });
+  });
+
+  describe('Edge cases and error handling', () => {
+    it('should handle results with null componentResults', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+      const db = await getDb();
+
+      // Insert result with grading_result but null componentResults
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "reason": "Test", "componentResults": null}'
+        )
+      `);
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      expect(filteredCount).toBe(0);
+      expect(testIndices).toEqual([]);
+    });
+
+    it('should handle results with multiple componentResults including human', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+      const db = await getDb();
+
+      // Result with multiple componentResults, one of which is human
+      await db.run(`
+        INSERT INTO eval_results (
+          id, eval_id, prompt_idx, test_idx, test_case, prompt, provider,
+          success, score, grading_result
+        ) VALUES (
+          'test1', '${eval_.id}', 0, 0, '{}', '{}', '{}', 1, 1.0,
+          '{"pass": true, "score": 1, "componentResults": [
+            {"assertion": {"type": "equals"}, "pass": true, "score": 1},
+            {"assertion": {"type": "human"}, "pass": true, "score": 1},
+            {"assertion": {"type": "contains"}, "pass": true, "score": 1}
+          ]}'
+        )
+      `);
+
+      const { testIndices, filteredCount } = await (eval_ as any).queryTestIndices({
+        filterMode: 'user-rated',
+      });
+
+      expect(filteredCount).toBe(1);
+      expect(testIndices).toEqual([0]);
+    });
+
+    it('should handle concurrent access to user-rated filter', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add some user-rated results
+      for (let i = 0; i < 10; i++) {
+        await eval_.addResult({
+          description: `test-${i}`,
+          promptIdx: 0,
+          testIdx: i,
+          testCase: { vars: { test: `value${i}` } },
+          promptId: 'test-prompt',
+          provider: { id: 'test-provider', label: 'test-label' },
+          prompt: { raw: 'Test prompt', label: 'Test prompt' },
+          vars: { test: `value${i}` },
+          response: { output: `Response ${i}` },
+          error: null,
+          failureReason: ResultFailureReason.NONE,
+          success: true,
+          score: 1,
+          latencyMs: 100,
+          gradingResult: createHumanRatedGradingResult(true, 1, `User rated ${i}`),
+          namedScores: {},
+          cost: 0.007,
+          metadata: {},
+        } as EvaluateResult);
+      }
+
+      // Run multiple queries concurrently
+      const promises = Array.from({ length: 5 }, () =>
+        (eval_ as any).queryTestIndices({ filterMode: 'user-rated' }),
+      );
+
+      const results = await Promise.all(promises);
+
+      // All concurrent queries should return the same results
+      const firstResult = results[0];
+      for (const result of results) {
+        expect(result.filteredCount).toBe(firstResult.filteredCount);
+        expect(result.testIndices).toEqual(firstResult.testIndices);
+      }
+    });
+
+    it('should verify SQL injection safety', async () => {
+      const eval_ = await EvalFactory.create({ numResults: 0 });
+
+      // Add a user-rated result
+      await eval_.addResult({
+        description: 'test-safe',
+        promptIdx: 0,
+        testIdx: 0,
+        testCase: { vars: { test: 'value' } },
+        promptId: 'test-prompt',
+        provider: { id: 'test-provider', label: 'test-label' },
+        prompt: { raw: 'Test prompt', label: 'Test prompt' },
+        vars: { test: 'value' },
+        response: { output: 'Safe response' },
+        error: null,
+        failureReason: ResultFailureReason.NONE,
+        success: true,
+        score: 1,
+        latencyMs: 100,
+        gradingResult: createHumanRatedGradingResult(true, 1, 'User rated'),
+        namedScores: {},
+        cost: 0.007,
+        metadata: {},
+      } as EvaluateResult);
+
+      // Try SQL injection in search query
+      const maliciousSearch = "'; DROP TABLE eval_results; --";
+      const { filteredCount } = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'user-rated',
+        searchQuery: maliciousSearch,
+      });
+
+      // Should not match the malicious search string
+      expect(filteredCount).toBe(0);
+
+      // Test that user-rated filter still works after attempted injection
+      const justUserRated = await queryTestIndicesOptimized(eval_.id, {
+        filterMode: 'user-rated',
+      });
+      expect(justUserRated.filteredCount).toBe(1);
+      expect(justUserRated.testIndices).toEqual([0]);
+    });
+  });
+});

test/models/modelAudit.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it, vi } from 'vitest';
+import ModelAudit from '../../src/models/modelAudit';
+
+import type { ModelAuditScanResults } from '../../src/types/modelAudit';
+
+// Mock the database
+vi.mock('../../src/database', () => ({
+  getDb: () => ({
+    insert: () => ({
+      values: () => ({
+        run: vi.fn().mockResolvedValue({}),
+      }),
+    }),
+    select: () => ({
+      from: () => ({
+        where: () => ({
+          get: vi.fn().mockResolvedValue(null),
+          orderBy: () => ({
+            get: vi.fn().mockResolvedValue(null),
+          }),
+        }),
+        orderBy: () => ({
+          all: vi.fn().mockResolvedValue([]),
+          limit: () => ({
+            all: vi.fn().mockResolvedValue([]),
+          }),
+        }),
+      }),
+    }),
+  }),
+}));
+
+describe('ModelAudit', () => {
+  describe('hasErrors logic', () => {
+    it('should set hasErrors to true when has_errors is true in results', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: true,
+        issues: [{ severity: 'warning', message: 'Test warning' }],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should set hasErrors to true when has_errors is false but critical issues exist', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false, // CLI tool incorrectly says no errors
+        issues: [
+          { severity: 'critical', message: 'Test critical issue' },
+          { severity: 'warning', message: 'Test warning' },
+        ],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should set hasErrors to true when has_errors is false but error issues exist', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false, // CLI tool incorrectly says no errors
+        issues: [
+          { severity: 'error', message: 'Test error issue' },
+          { severity: 'warning', message: 'Test warning' },
+        ],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should set hasErrors to true when warning findings exist', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false,
+        issues: [
+          { severity: 'warning', message: 'Test warning' },
+          { severity: 'info', message: 'Test info' },
+        ],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should set hasErrors to false when has_errors is false and no issues exist', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false,
+        issues: [],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(false);
+    });
+
+    it('should set hasErrors to true when failed checks exist without issue rows', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false,
+        failed_checks: 1,
+        issues: [],
+        checks: [{ name: 'pickle', status: 'failed', message: 'Check failed' }],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should handle missing issues array gracefully', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false,
+        // issues not provided
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(false);
+    });
+
+    it('should prioritize explicit hasErrors over computed value', () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false,
+        issues: [{ severity: 'critical', message: 'Test critical issue' }],
+      };
+
+      const audit = new ModelAudit({
+        modelPath: '/test/path',
+        results,
+        hasErrors: false, // Explicitly set to false
+      });
+
+      // Should still be false because explicit value takes precedence
+      expect(audit.hasErrors).toBe(false);
+    });
+  });
+
+  describe('ModelAudit.create', () => {
+    it('should properly set hasErrors in create method with critical issues', async () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false, // CLI tool incorrectly says no errors
+        issues: [{ severity: 'critical', message: 'Test critical issue' }],
+      };
+
+      const audit = await ModelAudit.create({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should properly set hasErrors in create method with error issues', async () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false, // CLI tool incorrectly says no errors
+        issues: [{ severity: 'error', message: 'Test error issue' }],
+      };
+
+      const audit = await ModelAudit.create({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+
+    it('should keep warning findings visible in create method', async () => {
+      const results: ModelAuditScanResults = {
+        has_errors: false,
+        issues: [{ severity: 'warning', message: 'Test warning' }],
+      };
+
+      const audit = await ModelAudit.create({
+        modelPath: '/test/path',
+        results,
+      });
+
+      expect(audit.hasErrors).toBe(true);
+    });
+  });
+});

test/models/prompt.test.ts

@@ -0,0 +1,189 @@
+import { describe, expect, it } from 'vitest';
+import { generateIdFromPrompt } from '../../src/models/prompt';
+import { sha256 } from '../../src/util/createHash';
+
+describe('generateIdFromPrompt', () => {
+  describe('priority order', () => {
+    it('uses label when provided (highest priority)', () => {
+      const result = generateIdFromPrompt({
+        id: 'my-id',
+        label: 'My Label',
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('My Label'));
+    });
+
+    it('uses id when label is empty string', () => {
+      const result = generateIdFromPrompt({
+        id: 'my-id',
+        label: '',
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('my-id'));
+    });
+
+    it('uses id when label is undefined', () => {
+      const result = generateIdFromPrompt({
+        id: 'my-id',
+        label: undefined,
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('my-id'));
+    });
+
+    it('uses id when label is not provided', () => {
+      const result = generateIdFromPrompt({
+        id: 'my-id',
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('my-id'));
+    });
+
+    it('falls back to raw when both label and id are empty', () => {
+      const result = generateIdFromPrompt({
+        id: '',
+        label: '',
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('My raw prompt'));
+    });
+
+    it('falls back to raw when both label and id are undefined', () => {
+      const result = generateIdFromPrompt({
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('My raw prompt'));
+    });
+  });
+
+  describe('raw content handling', () => {
+    it('hashes string raw content directly', () => {
+      const result = generateIdFromPrompt({
+        raw: 'Simple string prompt',
+      });
+      expect(result).toBe(sha256('Simple string prompt'));
+    });
+
+    it('stringifies object raw content before hashing', () => {
+      const rawObject = { role: 'user', content: 'Hello' };
+      const result = generateIdFromPrompt({
+        raw: rawObject,
+      });
+      expect(result).toBe(sha256(JSON.stringify(rawObject)));
+    });
+
+    it('stringifies array raw content before hashing', () => {
+      const rawArray = [
+        { role: 'system', content: 'You are helpful' },
+        { role: 'user', content: 'Hello' },
+      ];
+      const result = generateIdFromPrompt({
+        raw: rawArray,
+      });
+      expect(result).toBe(sha256(JSON.stringify(rawArray)));
+    });
+
+    it('handles empty string raw content', () => {
+      const result = generateIdFromPrompt({
+        raw: '',
+      });
+      expect(result).toBe(sha256(''));
+    });
+
+    it('handles empty object raw content', () => {
+      const result = generateIdFromPrompt({
+        raw: {},
+      });
+      expect(result).toBe(sha256('{}'));
+    });
+  });
+
+  describe('edge cases', () => {
+    it('treats whitespace-only label as truthy', () => {
+      const result = generateIdFromPrompt({
+        id: 'my-id',
+        label: '   ',
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('   '));
+    });
+
+    it('treats whitespace-only id as truthy', () => {
+      const result = generateIdFromPrompt({
+        id: '   ',
+        label: '',
+        raw: 'My raw prompt',
+      });
+      expect(result).toBe(sha256('   '));
+    });
+
+    it('produces consistent hashes for same input', () => {
+      const prompt = { label: 'Test', raw: 'Content' };
+      const result1 = generateIdFromPrompt(prompt);
+      const result2 = generateIdFromPrompt(prompt);
+      expect(result1).toBe(result2);
+    });
+
+    it('produces different hashes for different labels', () => {
+      const result1 = generateIdFromPrompt({ label: 'Label A', raw: 'Same content' });
+      const result2 = generateIdFromPrompt({ label: 'Label B', raw: 'Same content' });
+      expect(result1).not.toBe(result2);
+    });
+
+    it('produces different hashes for different raw content', () => {
+      const result1 = generateIdFromPrompt({ raw: 'Content A' });
+      const result2 = generateIdFromPrompt({ raw: 'Content B' });
+      expect(result1).not.toBe(result2);
+    });
+  });
+
+  describe('real-world scenarios', () => {
+    it('handles prompt with all properties', () => {
+      const result = generateIdFromPrompt({
+        id: 'prompt-123',
+        label: 'Customer Support Prompt',
+        raw: 'You are a helpful customer support agent. Answer: {{question}}',
+      });
+      expect(result).toBe(sha256('Customer Support Prompt'));
+    });
+
+    it('handles prompt with only id and raw (common case)', () => {
+      const result = generateIdFromPrompt({
+        id: 'prompt-123',
+        raw: 'Answer the question: {{question}}',
+      });
+      expect(result).toBe(sha256('prompt-123'));
+    });
+
+    it('handles prompt with template variables in raw', () => {
+      const result = generateIdFromPrompt({
+        raw: 'Translate {{text}} to {{language}}',
+      });
+      expect(result).toBe(sha256('Translate {{text}} to {{language}}'));
+    });
+
+    it('handles prompt with multiline raw content', () => {
+      const multilineRaw = `You are a helpful assistant.
+
+Please answer the following question:
+{{question}}
+
+Be concise and accurate.`;
+      const result = generateIdFromPrompt({ raw: multilineRaw });
+      expect(result).toBe(sha256(multilineRaw));
+    });
+
+    it('handles JSON-structured prompt', () => {
+      const jsonPrompt = {
+        raw: {
+          messages: [
+            { role: 'system', content: 'You are helpful' },
+            { role: 'user', content: '{{input}}' },
+          ],
+        },
+      };
+      const result = generateIdFromPrompt(jsonPrompt);
+      expect(result).toBe(sha256(JSON.stringify(jsonPrompt.raw)));
+    });
+  });
+});