promptfoo--promptfoo/test/evaluator/gradingConcurrency.test.ts

import './setup';

import { randomUUID } from 'crypto';

import { expect, it, vi } from 'vitest';
import { clearCache, getCache } from '../../src/cache';
import { evaluate, runEval } from '../../src/evaluator';
import Eval from '../../src/models/eval';
import {
  type ApiProvider,
  type RateLimitRegistryRef,
  ResultFailureReason,
  type TestSuite,
} from '../../src/types/index';
import { createEmptyTokenUsage } from '../../src/util/tokenUsageUtils';
import { toPrompt } from './helpers';
import { describeEvaluator } from './lifecycle';

describeEvaluator('evaluator grading concurrency', () => {
  it('schedules model-graded assertion provider calls through the rate limit registry', async () => {
    const abortController = new AbortController();
    const execute = vi.fn(async (_provider: ApiProvider, callFn: () => Promise<unknown>) =>
      callFn(),
    );
    const rateLimitRegistry = {
      execute,
      dispose: vi.fn(),
    } as RateLimitRegistryRef;
    const targetProvider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn().mockResolvedValue({
        output: 'Test response',
        tokenUsage: createEmptyTokenUsage(),
      }),
    };
    const gradingProvider: ApiProvider = {
      id: vi.fn().mockReturnValue('grading-provider'),
      callApi: vi.fn().mockResolvedValue({
        output: JSON.stringify({ pass: true, reason: 'Scheduled grading passed' }),
        tokenUsage: createEmptyTokenUsage(),
      }),
    };

    const results = await runEval({
      delay: 0,
      testIdx: 0,
      promptIdx: 0,
      repeatIndex: 0,
      isRedteam: false,
      provider: targetProvider,
      prompt: { raw: 'Test prompt', label: 'test-label' },
      test: {
        assert: [
          {
            type: 'llm-rubric',
            value: 'Output should be valid',
            provider: gradingProvider,
          },
        ],
      },
      conversations: {},
      registers: {},
      abortSignal: abortController.signal,
      rateLimitRegistry,
    });

    expect(results[0].success).toBe(true);
    expect(execute).toHaveBeenCalledTimes(2);
    expect(execute.mock.calls.map(([provider]) => provider.id())).toEqual([
      'target-provider',
      'grading-provider',
    ]);
    expect(gradingProvider.callApi).toHaveBeenCalledWith(
      expect.stringContaining('Output should be valid'),
      expect.objectContaining({
        prompt: { raw: expect.any(String), label: 'llm-rubric' },
        vars: expect.objectContaining({
          output: 'Test response',
          rubric: 'Output should be valid',
        }),
      }),
      { abortSignal: abortController.signal },
    );
  });

  it('groups model-graded assertion calls by provider id when maxConcurrency is 1', async () => {
    const callOrder: string[] = [];
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judgeOne: ApiProvider = {
      id: vi.fn().mockReturnValue('judge-one'),
      callApi: vi.fn(async () => {
        callOrder.push('judge-one');
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'judge one passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const judgeTwo: ApiProvider = {
      id: vi.fn().mockReturnValue('judge-two'),
      callApi: vi.fn(async () => {
        callOrder.push('judge-two');
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'judge two passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [
            { type: 'llm-rubric', value: 'Judge alpha one', provider: judgeOne },
            { type: 'llm-rubric', value: 'Judge alpha two', provider: judgeTwo },
          ],
        },
        {
          vars: { topic: 'beta' },
          assert: [
            { type: 'llm-rubric', value: 'Judge beta one', provider: judgeOne },
            { type: 'llm-rubric', value: 'Judge beta two', provider: judgeTwo },
          ],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    expect(summary.stats.successes).toBe(2);
    expect(callOrder).toEqual(['judge-one', 'judge-one', 'judge-two', 'judge-two']);
  });

  it('keeps model-graded assertions row-first when prompts use _conversation', async () => {
    const callOrder: string[] = [];
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => {
        const topic = prompt.endsWith('Current: alpha') ? 'alpha' : 'beta';
        callOrder.push(`target:${topic}`);
        return {
          output: `Target output for ${topic}`,
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => {
        callOrder.push('judge');
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'judge passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [
        toPrompt(
          '{% for turn in _conversation %}{{ turn.input }} => {{ turn.output }}\n{% endfor %}Current: {{topic}}',
        ),
      ],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
        {
          vars: { topic: 'beta' },
          assert: [{ type: 'llm-rubric', value: 'Judge beta', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    expect(summary.stats.successes).toBe(2);
    expect(callOrder).toEqual(['target:alpha', 'judge', 'target:beta', 'judge']);
  });

  it('records deferred model-graded provider failures as row errors when maxConcurrency is 1', async () => {
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async (prompt: string) => {
        if (prompt.includes('Judge alpha')) {
          throw new Error('grader exploded');
        }
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'judge passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
        {
          vars: { topic: 'beta' },
          assert: [{ type: 'llm-rubric', value: 'Judge beta', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    const failedResult = summary.results.find((result) => result.vars.topic === 'alpha');
    expect(summary.stats.errors).toBe(1);
    expect(summary.stats.successes).toBe(1);
    expect(summary.results).toHaveLength(2);
    expect(failedResult?.failureReason).toBe(ResultFailureReason.ERROR);
    expect(failedResult?.error).toContain('grader exploded');
  });

  it('stops grouped serial evals after a non-transient target status', async () => {
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        metadata: {
          http: {
            status: 403,
            statusText: 'Forbidden',
          },
        },
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => ({
        output: JSON.stringify({ pass: true, score: 1, reason: 'judge passed' }),
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
        {
          vars: { topic: 'beta' },
          assert: [{ type: 'llm-rubric', value: 'Judge beta', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    expect(provider.callApi).toHaveBeenCalledTimes(1);
    expect(judge.callApi).toHaveBeenCalledTimes(1);
    expect(summary.results).toHaveLength(1);
    expect(summary.results[0].vars.topic).toBe('alpha');
  });

  it('groups model-graded assert-set children by provider id when maxConcurrency is 1', async () => {
    const callOrder: string[] = [];
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judgeOne: ApiProvider = {
      id: vi.fn().mockReturnValue('judge-one'),
      callApi: vi.fn(async () => {
        callOrder.push('judge-one');
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'judge one passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const judgeTwo: ApiProvider = {
      id: vi.fn().mockReturnValue('judge-two'),
      callApi: vi.fn(async () => {
        callOrder.push('judge-two');
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'judge two passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [
            {
              type: 'assert-set',
              assert: [
                { type: 'llm-rubric', value: 'Judge alpha one', provider: judgeOne },
                { type: 'llm-rubric', value: 'Judge alpha two', provider: judgeTwo },
              ],
            },
          ],
        },
        {
          vars: { topic: 'beta' },
          assert: [
            {
              type: 'assert-set',
              assert: [
                { type: 'llm-rubric', value: 'Judge beta one', provider: judgeOne },
                { type: 'llm-rubric', value: 'Judge beta two', provider: judgeTwo },
              ],
            },
          ],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    expect(summary.stats.successes).toBe(2);
    expect(callOrder).toEqual(['judge-one', 'judge-one', 'judge-two', 'judge-two']);
  });

  it('keeps multi-call model-graded assertions grouped by provider id when maxConcurrency is 1', async () => {
    const callOrder: string[] = [];
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const createJudge = (id: string): ApiProvider => ({
      id: vi.fn().mockReturnValue(id),
      callApi: vi.fn(async (_prompt: string, context?: { prompt?: { label?: string } }) => {
        const label = context?.prompt?.label ?? 'unknown';
        callOrder.push(`${id}:${label}`);

        return {
          output:
            label === 'g-eval-steps'
              ? JSON.stringify({ steps: ['Check the answer'] })
              : JSON.stringify({ score: 10, reason: 'passed' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    });
    const judgeOne = createJudge('judge-one');
    const judgeTwo = createJudge('judge-two');
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [
            { type: 'g-eval', value: 'Judge alpha one', provider: judgeOne },
            { type: 'g-eval', value: 'Judge alpha two', provider: judgeTwo },
          ],
        },
        {
          vars: { topic: 'beta' },
          assert: [
            { type: 'g-eval', value: 'Judge beta one', provider: judgeOne },
            { type: 'g-eval', value: 'Judge beta two', provider: judgeTwo },
          ],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    expect(summary.stats.successes).toBe(2);
    expect(callOrder).toEqual([
      'judge-one:g-eval-steps',
      'judge-one:g-eval-steps',
      'judge-one:g-eval',
      'judge-one:g-eval',
      'judge-two:g-eval-steps',
      'judge-two:g-eval-steps',
      'judge-two:g-eval',
      'judge-two:g-eval',
    ]);
  });

  it('isolates deferred grading cache entries by repeat index', async () => {
    await clearCache();

    const provider: ApiProvider = {
      id: () => 'mock-provider',
      callApi: vi.fn().mockResolvedValue({
        cached: false,
        output: 'target output',
        tokenUsage: createEmptyTokenUsage(),
      }),
    };

    let gradingCacheMissCount = 0;
    const gradingProvider: ApiProvider = {
      id: () => 'grading-provider',
      callApi: vi.fn().mockImplementation(async () => {
        const cache = getCache();
        const cachedOutput = await cache.get<string>('deferred-grading-key');
        if (cachedOutput) {
          return {
            cached: true,
            output: cachedOutput,
            tokenUsage: createEmptyTokenUsage(),
          };
        }

        gradingCacheMissCount += 1;
        const output = JSON.stringify({
          pass: true,
          reason: `grader-repeat-${gradingCacheMissCount}`,
          score: 1,
        });
        await cache.set('deferred-grading-key', output);
        return {
          cached: false,
          output,
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };

    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt')],
      tests: [
        {
          assert: [{ type: 'llm-rubric', value: 'Output should pass', provider: gradingProvider }],
        },
      ],
    };

    const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, firstEval, { maxConcurrency: 1, repeat: 2 });

    expect(gradingCacheMissCount).toBe(2);

    const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, secondEval, { maxConcurrency: 1, repeat: 2 });

    expect(gradingCacheMissCount).toBe(2);
    expect(gradingProvider.callApi).toHaveBeenCalledTimes(4);
  });

  it('serializes async assert-set judges when provider call queue is active', async () => {
    // Regression: without forcing assertion concurrency to 1 when the grouping
    // queue is active, async graders inside an assert-set would interleave
    // judges across rows (judge-one row1, judge-two row1, judge-one row2, ...)
    // and defeat the single-judge-drain guarantee.
    const callOrder: string[] = [];
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const makeAsyncJudge = (id: string): ApiProvider => ({
      id: vi.fn().mockReturnValue(id),
      callApi: vi.fn(async () => {
        // Intentionally yield through both microtask and macrotask queues so
        // this regression test can expose the cross-row interleaving that
        // assertion concurrency caused.
        await Promise.resolve();
        await new Promise<void>((resolve) => setImmediate(resolve));
        callOrder.push(id);
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: `${id} passed` }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    });
    const judgeOne = makeAsyncJudge('judge-one');
    const judgeTwo = makeAsyncJudge('judge-two');
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [
            {
              type: 'assert-set',
              assert: [
                { type: 'llm-rubric', value: 'Judge alpha one', provider: judgeOne },
                { type: 'llm-rubric', value: 'Judge alpha two', provider: judgeTwo },
              ],
            },
          ],
        },
        {
          vars: { topic: 'beta' },
          assert: [
            {
              type: 'assert-set',
              assert: [
                { type: 'llm-rubric', value: 'Judge beta one', provider: judgeOne },
                { type: 'llm-rubric', value: 'Judge beta two', provider: judgeTwo },
              ],
            },
          ],
        },
        {
          vars: { topic: 'gamma' },
          assert: [
            {
              type: 'assert-set',
              assert: [
                { type: 'llm-rubric', value: 'Judge gamma one', provider: judgeOne },
                { type: 'llm-rubric', value: 'Judge gamma two', provider: judgeTwo },
              ],
            },
          ],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });
    const summary = await evalRecord.toEvaluateSummary();

    expect(summary.stats.successes).toBe(3);
    expect(callOrder).toEqual([
      'judge-one',
      'judge-one',
      'judge-one',
      'judge-two',
      'judge-two',
      'judge-two',
    ]);
  });

  it('does not log error-level message when deferred grading is aborted', async () => {
    const { default: logger } = await import('../../src/logger');
    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);

    const abortController = new AbortController();
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => {
        abortController.abort();
        const abortError = new Error('Aborted');
        abortError.name = 'AbortError';
        throw abortError;
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, {
      maxConcurrency: 1,
      abortSignal: abortController.signal,
    });

    const gradingErrorLogs = errorSpy.mock.calls.filter(
      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
    );
    expect(gradingErrorLogs).toHaveLength(0);

    const failedResult = (await evalRecord.toEvaluateSummary()).results.find(
      (result) => result.vars.topic === 'alpha',
    );
    expect(failedResult?.error).toMatch(/^Aborted: /);

    errorSpy.mockRestore();
  });

  it('suppresses the error log when deferred grading throws AbortException under abort', async () => {
    const { default: logger } = await import('../../src/logger');
    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);

    const abortController = new AbortController();
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => {
        abortController.abort();
        const abortException = new Error('Python provider cancelled');
        abortException.name = 'AbortException';
        throw abortException;
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, {
      maxConcurrency: 1,
      abortSignal: abortController.signal,
    });

    const gradingErrorLogs = errorSpy.mock.calls.filter(
      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
    );
    expect(gradingErrorLogs).toHaveLength(0);

    errorSpy.mockRestore();
  });

  it('still logs error-level when abort-shaped error fires WITHOUT an aborted signal', async () => {
    // Regression guard: third-party SDKs sometimes surface unrelated cancellation
    // as AbortError. If the evaluator's abort signal is NOT tripped, these are
    // real bugs and must still surface at error level.
    const { default: logger } = await import('../../src/logger');
    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);

    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => {
        const sdkAbortLookalike = new Error('fetch aborted by keepalive');
        sdkAbortLookalike.name = 'AbortError';
        throw sdkAbortLookalike;
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });

    const gradingErrorLogs = errorSpy.mock.calls.filter(
      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
    );
    expect(gradingErrorLogs.length).toBeGreaterThanOrEqual(1);

    const summary = await evalRecord.toEvaluateSummary();
    expect(summary.stats.errors).toBe(1);
    expect(summary.results[0].failureReason).toBe(ResultFailureReason.ERROR);
    expect(summary.results[0].error).not.toMatch(/^Aborted: /);

    errorSpy.mockRestore();
  });

  it('still logs error when a non-abort-shaped error fires during an unrelated abort', async () => {
    // Regression guard: if abort is in flight but the caught error is a real
    // bug (SyntaxError, TypeError, etc.), we must not silently suppress it.
    const { default: logger } = await import('../../src/logger');
    const errorSpy = vi.spyOn(logger, 'error').mockImplementation(() => logger);

    const abortController = new AbortController();
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => {
        abortController.abort();
        throw new SyntaxError('Unexpected token in grader output');
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, {
      maxConcurrency: 1,
      abortSignal: abortController.signal,
    });

    const gradingErrorLogs = errorSpy.mock.calls.filter(
      ([message]) => typeof message === 'string' && message.includes('Assertion grading failed'),
    );
    expect(gradingErrorLogs.length).toBeGreaterThanOrEqual(1);

    errorSpy.mockRestore();
  });

  it('logs grouping-active info when serial eval contains model-graded assertions', async () => {
    const { default: logger } = await import('../../src/logger');
    const infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);

    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => ({
        output: JSON.stringify({ pass: true, score: 1, reason: 'ok' }),
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });

    const groupingLogs = infoSpy.mock.calls.filter(
      ([message]) =>
        typeof message === 'string' &&
        message.includes('Grouping model-graded assertions by provider'),
    );
    expect(groupingLogs).toHaveLength(1);

    infoSpy.mockRestore();
  });

  it('does not log grouping info when there are no model-graded assertions', async () => {
    const { default: logger } = await import('../../src/logger');
    const infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);

    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt {{topic}}')],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'contains', value: 'Target' }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });

    const groupingLogs = infoSpy.mock.calls.filter(
      ([message]) =>
        typeof message === 'string' &&
        (message.includes('Grouping model-graded assertions by provider') ||
          message.includes('Serial grading grouping disabled')),
    );
    expect(groupingLogs).toHaveLength(0);

    infoSpy.mockRestore();
  });

  it('logs grouping-disabled reason when conversation var forces per-row ordering', async () => {
    const { default: logger } = await import('../../src/logger');
    const infoSpy = vi.spyOn(logger, 'info').mockImplementation(() => logger);

    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async () => ({
        output: 'Target output',
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => ({
        output: JSON.stringify({ pass: true, score: 1, reason: 'ok' }),
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [
        toPrompt('{% for turn in _conversation %}{{ turn.input }}{% endfor %}Current: {{topic}}'),
      ],
      tests: [
        {
          vars: { topic: 'alpha' },
          assert: [{ type: 'llm-rubric', value: 'Judge alpha', provider: judge }],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    await evaluate(testSuite, evalRecord, { maxConcurrency: 1 });

    const disabledLogs = infoSpy.mock.calls.filter(
      ([message]) =>
        typeof message === 'string' &&
        message.includes('Serial grading grouping disabled') &&
        message.includes('conversation variables'),
    );
    expect(disabledLogs).toHaveLength(1);

    infoSpy.mockRestore();
  });

  it('keeps parallel assertion dispatch when the grouping queue is NOT active', async () => {
    // Regression guard for the `? 1 : ASSERTIONS_MAX_CONCURRENCY` ternary in
    // runAssertions. Without the queue present (non-deferred concurrent eval),
    // per-test assertions must still fan out so we don't silently 3x-throttle
    // normal eval users.
    let inFlight = 0;
    let maxInFlight = 0;
    const provider: ApiProvider = {
      id: vi.fn().mockReturnValue('target-provider'),
      callApi: vi.fn(async (prompt: string) => ({
        output: `Target output for ${prompt}`,
        tokenUsage: createEmptyTokenUsage(),
      })),
    };
    // Concurrency barrier: hold every in-flight judge until at least two have
    // started, then release all of them together. This makes maxInFlight
    // observation independent of wall-clock pacing.
    let releaseHold: (() => void) | undefined;
    const holdPromise = new Promise<void>((resolve) => {
      releaseHold = resolve;
    });
    const judge: ApiProvider = {
      id: vi.fn().mockReturnValue('judge'),
      callApi: vi.fn(async () => {
        inFlight += 1;
        maxInFlight = Math.max(maxInFlight, inFlight);
        if (inFlight >= 2) {
          releaseHold?.();
        }
        let resolveFallback!: () => void;
        const fallback = new Promise<void>((r) => {
          resolveFallback = r;
        });
        const fallbackHandle = setTimeout(() => resolveFallback(), 100);
        try {
          await Promise.race([holdPromise, fallback]);
        } finally {
          clearTimeout(fallbackHandle);
        }
        inFlight -= 1;
        return {
          output: JSON.stringify({ pass: true, score: 1, reason: 'ok' }),
          tokenUsage: createEmptyTokenUsage(),
        };
      }),
    };
    const testSuite: TestSuite = {
      providers: [provider],
      prompts: [toPrompt('Test prompt')],
      tests: [
        {
          assert: [
            { type: 'llm-rubric', value: 'Judge one', provider: judge },
            { type: 'llm-rubric', value: 'Judge two', provider: judge },
            { type: 'llm-rubric', value: 'Judge three', provider: judge },
          ],
        },
      ],
    };

    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
    // maxConcurrency > 1 takes the non-grouped path where the queue is absent.
    await evaluate(testSuite, evalRecord, { maxConcurrency: 5 });

    expect(inFlight).toBe(0);
    expect(maxInFlight).toBeGreaterThanOrEqual(2);
  });
});