chore: import upstream snapshot with attribution

test/tracing/evaluatorTracing.test.ts

@@ -0,0 +1,828 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import logger from '../../src/logger';
+import {
+  generateSpanId,
+  generateTraceContextIfNeeded,
+  generateTraceId,
+  generateTraceparent,
+  isOtlpReceiverStarted,
+  isTracingEnabled,
+  resetTracingState,
+  startOtlpReceiverIfNeeded,
+  stopOtlpReceiverIfNeeded,
+} from '../../src/tracing/evaluatorTracing';
+import { getTraceStore } from '../../src/tracing/store';
+import { mockProcessEnv } from '../util/utils';
+
+import type { TestCase, TestSuite } from '../../src/types/index';
+
+const mockStartOTLPReceiver = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+const mockStopOTLPReceiver = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+const mockUpdateOTLPReceiverOptions = vi.hoisted(() => vi.fn());
+const mockRegisterOTLPReceiverTracePolicy = vi.hoisted(() => vi.fn());
+const mockCreateTrace = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
+
+// Mock the logger
+vi.mock('../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+// Mock the trace store
+vi.mock('../../src/tracing/store', () => ({
+  getTraceStore: vi.fn(() => ({
+    createTrace: mockCreateTrace,
+  })),
+}));
+
+vi.mock('../../src/tracing/otlpReceiver', () => ({
+  startOTLPReceiver: mockStartOTLPReceiver,
+  stopOTLPReceiver: mockStopOTLPReceiver,
+  updateOTLPReceiverOptions: mockUpdateOTLPReceiverOptions,
+  registerOTLPReceiverTracePolicy: mockRegisterOTLPReceiverTracePolicy,
+}));
+
+describe('evaluatorTracing', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    mockStartOTLPReceiver.mockReset();
+    mockStartOTLPReceiver.mockResolvedValue(undefined);
+    mockStopOTLPReceiver.mockReset();
+    mockStopOTLPReceiver.mockResolvedValue(undefined);
+    mockUpdateOTLPReceiverOptions.mockReset();
+    mockRegisterOTLPReceiverTracePolicy.mockReset();
+    mockCreateTrace.mockReset();
+    mockCreateTrace.mockResolvedValue(undefined);
+    vi.mocked(getTraceStore).mockReturnValue({
+      createTrace: mockCreateTrace,
+    } as unknown as ReturnType<typeof getTraceStore>);
+    resetTracingState();
+    // Reset environment variables
+    mockProcessEnv({ PROMPTFOO_TRACING_ENABLED: undefined });
+  });
+
+  describe('generateTraceId', () => {
+    it('should generate a 32-character hex string', () => {
+      const traceId = generateTraceId();
+      expect(traceId).toMatch(/^[a-f0-9]{32}$/);
+      expect(traceId).toHaveLength(32);
+    });
+
+    it('should generate unique IDs', () => {
+      const id1 = generateTraceId();
+      const id2 = generateTraceId();
+      expect(id1).not.toBe(id2);
+    });
+  });
+
+  describe('generateSpanId', () => {
+    it('should generate a 16-character hex string', () => {
+      const spanId = generateSpanId();
+      expect(spanId).toMatch(/^[a-f0-9]{16}$/);
+      expect(spanId).toHaveLength(16);
+    });
+
+    it('should generate unique IDs', () => {
+      const id1 = generateSpanId();
+      const id2 = generateSpanId();
+      expect(id1).not.toBe(id2);
+    });
+  });
+
+  describe('generateTraceparent', () => {
+    it('should generate valid W3C Trace Context format with sampled flag', () => {
+      const traceId = 'a'.repeat(32);
+      const spanId = 'b'.repeat(16);
+      const traceparent = generateTraceparent(traceId, spanId, true);
+      expect(traceparent).toBe(`00-${traceId}-${spanId}-01`);
+    });
+
+    it('should generate valid W3C Trace Context format without sampled flag', () => {
+      const traceId = 'a'.repeat(32);
+      const spanId = 'b'.repeat(16);
+      const traceparent = generateTraceparent(traceId, spanId, false);
+      expect(traceparent).toBe(`00-${traceId}-${spanId}-00`);
+    });
+
+    it('should default to sampled=true', () => {
+      const traceId = 'a'.repeat(32);
+      const spanId = 'b'.repeat(16);
+      const traceparent = generateTraceparent(traceId, spanId);
+      expect(traceparent).toBe(`00-${traceId}-${spanId}-01`);
+    });
+  });
+
+  describe('generateTraceContextIfNeeded', () => {
+    it('should return null when tracing is not enabled', async () => {
+      const test: TestCase = {
+        vars: { foo: 'bar' },
+      };
+      const result = await generateTraceContextIfNeeded(test, {}, 0, 0);
+      expect(result).toBeNull();
+    });
+
+    it('should generate trace context when tracing is enabled via metadata', async () => {
+      const test: TestCase = {
+        vars: { foo: 'bar' },
+        metadata: {
+          tracingEnabled: true,
+          evaluationId: 'eval-123',
+          testCaseId: 'test-456',
+        },
+      };
+      const result = await generateTraceContextIfNeeded(test, {}, 0, 0);
+
+      expect(result).not.toBeNull();
+      expect(result!.traceparent).toMatch(/^00-[a-f0-9]{32}-[a-f0-9]{16}-01$/);
+      expect(result!.evaluationId).toBe('eval-123');
+      expect(result!.testCaseId).toBe('test-456');
+    });
+
+    it('should generate trace context when tracing is enabled via environment', async () => {
+      mockProcessEnv({ PROMPTFOO_TRACING_ENABLED: 'true' });
+      const test: TestCase = {
+        vars: { foo: 'bar' },
+      };
+      const result = await generateTraceContextIfNeeded(test, {}, 5, 10);
+
+      expect(result).not.toBeNull();
+      expect(result!.traceparent).toMatch(/^00-[a-f0-9]{32}-[a-f0-9]{16}-01$/);
+      expect(result!.evaluationId).toMatch(/^eval-/);
+      expect(result!.testCaseId).toBe('5-10');
+    });
+
+    it('should generate trace context when tracing is enabled via YAML config', async () => {
+      const test: TestCase = {
+        vars: { foo: 'bar' },
+      };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+        },
+      } as unknown as TestSuite;
+
+      const result = await generateTraceContextIfNeeded(test, {}, 3, 7, testSuite);
+
+      expect(result).not.toBeNull();
+      expect(result!.traceparent).toMatch(/^00-[a-f0-9]{32}-[a-f0-9]{16}-01$/);
+      expect(result!.evaluationId).toMatch(/^eval-/);
+      expect(result!.testCaseId).toBe('3-7');
+    });
+
+    it('should persist command tool-name overrides in trace metadata', async () => {
+      const test: TestCase = {
+        vars: { foo: 'bar' },
+      };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          commandToolNames: ['bash'],
+        },
+      } as unknown as TestSuite;
+      const traceStoreModule = await import('../../src/tracing/store');
+      const getTraceStoreSpy = vi.spyOn(traceStoreModule, 'getTraceStore').mockReturnValue({
+        createTrace: mockCreateTrace,
+      } as unknown as ReturnType<typeof traceStoreModule.getTraceStore>);
+
+      try {
+        await generateTraceContextIfNeeded(test, {}, 2, 4, testSuite);
+
+        expect(mockCreateTrace).toHaveBeenCalledWith(
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              commandToolNames: ['bash'],
+            }),
+          }),
+        );
+      } finally {
+        getTraceStoreSpy.mockRestore();
+      }
+    });
+
+    it('should persist OTLP redaction config in trace metadata', async () => {
+      const test: TestCase = {
+        vars: { foo: 'bar' },
+      };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: {
+            http: {
+              enabled: true,
+              redactAttributes: ['authorization'],
+            },
+          },
+        },
+      } as unknown as TestSuite;
+
+      await generateTraceContextIfNeeded(test, {}, 2, 4, testSuite);
+
+      expect(mockCreateTrace).toHaveBeenCalledWith(
+        expect.objectContaining({
+          metadata: expect.objectContaining({
+            otlpHttpRedactAttributes: ['authorization'],
+          }),
+        }),
+      );
+    });
+  });
+
+  describe('isTracingEnabled', () => {
+    it('should return false when no tracing is configured', () => {
+      const test: TestCase = { vars: {} };
+      expect(isTracingEnabled(test)).toBe(false);
+    });
+
+    it('should return true when test metadata has tracingEnabled', () => {
+      const test: TestCase = {
+        vars: {},
+        metadata: { tracingEnabled: true },
+      };
+      expect(isTracingEnabled(test)).toBe(true);
+    });
+
+    it('should return true when environment variable is set', () => {
+      mockProcessEnv({ PROMPTFOO_TRACING_ENABLED: 'true' });
+      const test: TestCase = { vars: {} };
+      expect(isTracingEnabled(test)).toBe(true);
+    });
+
+    it('should return true when testSuite.tracing.enabled is true', () => {
+      const test: TestCase = { vars: {} };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: { enabled: true },
+      } as unknown as TestSuite;
+      expect(isTracingEnabled(test, testSuite)).toBe(true);
+    });
+
+    it('should return false when testSuite.tracing.enabled is false', () => {
+      const test: TestCase = { vars: {} };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: { enabled: false },
+      } as unknown as TestSuite;
+      expect(isTracingEnabled(test, testSuite)).toBe(false);
+    });
+
+    it('should return true when any source enables tracing', () => {
+      const test: TestCase = {
+        vars: {},
+        metadata: { tracingEnabled: false },
+      };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: { enabled: true },
+      } as unknown as TestSuite;
+      // testSuite enables tracing even though metadata doesn't
+      expect(isTracingEnabled(test, testSuite)).toBe(true);
+    });
+  });
+
+  describe('isOtlpReceiverStarted', () => {
+    it('should return false initially', () => {
+      expect(isOtlpReceiverStarted()).toBe(false);
+    });
+  });
+
+  describe('startOtlpReceiverIfNeeded', () => {
+    it('should pass tracing configuration to the logger as sanitizable context', async () => {
+      const secret = 'Bearer secret-token';
+      const tracing = {
+        enabled: true,
+        forwarding: {
+          enabled: true,
+          endpoint: 'https://example.com/v1/traces',
+          headers: { authorization: secret },
+        },
+      };
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing,
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(logger.debug).toHaveBeenCalledWith(
+        '[EvaluatorTracing] Checking tracing configuration',
+        {
+          tracing,
+          testSuiteKeys: ['providers', 'prompts', 'tracing'],
+        },
+      );
+      expect(
+        vi.mocked(logger.debug).mock.calls.some(([message]) => String(message).includes(secret)),
+      ).toBe(false);
+    });
+
+    it('should pass configured acceptFormats to the OTLP receiver', async () => {
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '0.0.0.0',
+              acceptFormats: ['protobuf'],
+            },
+          },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledWith(4318, '0.0.0.0', ['protobuf'], {
+        commandToolNames: undefined,
+        redactAttributes: undefined,
+      });
+    });
+
+    it('should swallow receiver start errors by default', async () => {
+      mockStartOTLPReceiver.mockRejectedValueOnce(new Error('EADDRINUSE: port 4318'));
+
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite;
+
+      await expect(startOtlpReceiverIfNeeded(testSuite)).resolves.toBe(false);
+      expect(isOtlpReceiverStarted()).toBe(false);
+    });
+
+    it('should rethrow receiver start errors when failOnReceiverStartFailure is true', async () => {
+      mockStartOTLPReceiver.mockRejectedValueOnce(new Error('EADDRINUSE: port 4318'));
+
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          failOnReceiverStartFailure: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite;
+
+      await expect(startOtlpReceiverIfNeeded(testSuite)).rejects.toThrow(
+        /Failed to start OTLP tracing receiver: EADDRINUSE/,
+      );
+    });
+
+    it('should prune the trace store when storage.retentionDays is set', async () => {
+      const deleteOldTraces = vi.fn().mockResolvedValue(undefined);
+      const { getTraceStore } = await import('../../src/tracing/store');
+      vi.mocked(getTraceStore).mockReturnValue({
+        deleteOldTraces,
+      } as unknown as ReturnType<typeof getTraceStore>);
+
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+          storage: { type: 'sqlite', retentionDays: 7 },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(deleteOldTraces).toHaveBeenCalledWith(7);
+    });
+
+    it('should prune the trace store without requiring an OTLP HTTP receiver', async () => {
+      const deleteOldTraces = vi.fn().mockResolvedValue(undefined);
+      vi.mocked(getTraceStore).mockReturnValue({
+        deleteOldTraces,
+      } as unknown as ReturnType<typeof getTraceStore>);
+
+      await startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          storage: { type: 'sqlite', retentionDays: 7 },
+        },
+      } as unknown as TestSuite);
+
+      expect(deleteOldTraces).toHaveBeenCalledWith(7);
+      expect(mockStartOTLPReceiver).not.toHaveBeenCalled();
+    });
+
+    it('should prune the trace store when tracing is enabled by environment', async () => {
+      const deleteOldTraces = vi.fn().mockResolvedValue(undefined);
+      vi.mocked(getTraceStore).mockReturnValue({
+        deleteOldTraces,
+      } as unknown as ReturnType<typeof getTraceStore>);
+      mockProcessEnv({ PROMPTFOO_TRACING_ENABLED: 'true' });
+
+      await startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          storage: { type: 'sqlite', retentionDays: 7 },
+        },
+      } as unknown as TestSuite);
+
+      expect(deleteOldTraces).toHaveBeenCalledWith(7);
+      expect(mockStartOTLPReceiver).not.toHaveBeenCalled();
+    });
+
+    it('should not prune when retentionDays is omitted', async () => {
+      const deleteOldTraces = vi.fn().mockResolvedValue(undefined);
+      const { getTraceStore } = await import('../../src/tracing/store');
+      vi.mocked(getTraceStore).mockReturnValue({
+        deleteOldTraces,
+      } as unknown as ReturnType<typeof getTraceStore>);
+
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(deleteOldTraces).not.toHaveBeenCalled();
+    });
+
+    it('should pass redactAttributes to the OTLP receiver', async () => {
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+              redactAttributes: ['tool.arguments', 'authorization'],
+            },
+          },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledWith(4318, '127.0.0.1', ['json', 'protobuf'], {
+        commandToolNames: undefined,
+        redactAttributes: ['tool.arguments', 'authorization'],
+      });
+    });
+
+    it('should pass commandToolNames to the OTLP receiver', async () => {
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          commandToolNames: ['bash', 'shell'],
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+            },
+          },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledWith(4318, '127.0.0.1', ['json', 'protobuf'], {
+        commandToolNames: ['bash', 'shell'],
+        redactAttributes: undefined,
+      });
+    });
+
+    it('should pass the current evaluation policy when starting the OTLP receiver', async () => {
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          commandToolNames: ['bash'],
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+              redactAttributes: ['authorization'],
+            },
+          },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite, 'eval-current');
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledWith(4318, '127.0.0.1', ['json', 'protobuf'], {
+        commandToolNames: ['bash'],
+        redactAttributes: ['authorization'],
+        tracePolicy: {
+          evaluationId: 'eval-current',
+          commandToolNames: ['bash'],
+          redactAttributes: ['authorization'],
+        },
+      });
+    });
+
+    it('should default acceptFormats to both when omitted', async () => {
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '0.0.0.0',
+            },
+          },
+        },
+      } as unknown as TestSuite;
+
+      await startOtlpReceiverIfNeeded(testSuite);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledWith(4318, '0.0.0.0', ['json', 'protobuf'], {
+        commandToolNames: undefined,
+        redactAttributes: undefined,
+      });
+    });
+
+    it('should prune the trace store when tracing is already running', async () => {
+      const deleteOldTraces = vi.fn().mockResolvedValue(undefined);
+      const { getTraceStore } = await import('../../src/tracing/store');
+      vi.mocked(getTraceStore).mockReturnValue({
+        deleteOldTraces,
+      } as unknown as ReturnType<typeof getTraceStore>);
+
+      await startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite);
+
+      await startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+            },
+          },
+          storage: { type: 'sqlite', retentionDays: 7 },
+        },
+      } as unknown as TestSuite);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1);
+      expect(deleteOldTraces).toHaveBeenCalledWith(7);
+    });
+
+    it('should not mutate receiver defaults when another traced evaluation is already running', async () => {
+      await startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite);
+
+      await startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          commandToolNames: ['bash'],
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+              acceptFormats: ['json'],
+              redactAttributes: ['authorization'],
+            },
+          },
+        },
+      } as unknown as TestSuite);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1);
+      expect(mockUpdateOTLPReceiverOptions).not.toHaveBeenCalled();
+    });
+
+    it('should register the current evaluation policy when reusing an active receiver', async () => {
+      await startOtlpReceiverIfNeeded(
+        {
+          providers: [],
+          prompts: [],
+          tracing: {
+            enabled: true,
+            otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+          },
+        } as unknown as TestSuite,
+        'eval-a',
+      );
+
+      await startOtlpReceiverIfNeeded(
+        {
+          providers: [],
+          prompts: [],
+          tracing: {
+            enabled: true,
+            commandToolNames: ['bash'],
+            otlp: {
+              http: {
+                enabled: true,
+                port: 4318,
+                host: '127.0.0.1',
+                redactAttributes: ['authorization'],
+              },
+            },
+          },
+        } as unknown as TestSuite,
+        'eval-b',
+      );
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1);
+      expect(mockRegisterOTLPReceiverTracePolicy).toHaveBeenCalledWith({
+        evaluationId: 'eval-b',
+        commandToolNames: ['bash'],
+        redactAttributes: ['authorization'],
+      });
+      expect(mockUpdateOTLPReceiverOptions).not.toHaveBeenCalled();
+    });
+
+    it('should preserve the first receiver defaults during overlapping startup', async () => {
+      let resolveStart!: () => void;
+      mockStartOTLPReceiver.mockImplementationOnce(
+        () =>
+          new Promise<void>((resolve) => {
+            resolveStart = resolve;
+          }),
+      );
+
+      const firstStart = startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          commandToolNames: ['bash'],
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+              acceptFormats: ['json'],
+              redactAttributes: ['authorization'],
+            },
+          },
+        },
+      } as unknown as TestSuite);
+      const secondStart = startOtlpReceiverIfNeeded({
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          commandToolNames: ['shell'],
+          otlp: {
+            http: {
+              enabled: true,
+              port: 4318,
+              host: '127.0.0.1',
+              acceptFormats: ['protobuf'],
+              redactAttributes: ['cookie'],
+            },
+          },
+        },
+      } as unknown as TestSuite);
+
+      await vi.waitFor(() => expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1));
+      resolveStart();
+      await Promise.all([firstStart, secondStart]);
+
+      expect(mockStartOTLPReceiver).toHaveBeenCalledWith(4318, '127.0.0.1', ['json'], {
+        commandToolNames: ['bash'],
+        redactAttributes: ['authorization'],
+      });
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1);
+      expect(mockUpdateOTLPReceiverOptions).not.toHaveBeenCalled();
+    });
+
+    it('should keep the receiver live until every overlapping evaluation releases its lease', async () => {
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite;
+
+      const firstLease = await startOtlpReceiverIfNeeded(testSuite);
+      const secondLease = await startOtlpReceiverIfNeeded(testSuite);
+
+      await stopOtlpReceiverIfNeeded(firstLease);
+      expect(mockStopOTLPReceiver).not.toHaveBeenCalled();
+      expect(isOtlpReceiverStarted()).toBe(true);
+
+      await stopOtlpReceiverIfNeeded(secondLease);
+      expect(mockStopOTLPReceiver).toHaveBeenCalledTimes(1);
+      expect(isOtlpReceiverStarted()).toBe(false);
+    });
+
+    it('should wait for an in-flight shutdown before starting a new receiver', async () => {
+      let resolveStop!: () => void;
+      mockStopOTLPReceiver.mockImplementationOnce(
+        () =>
+          new Promise<void>((resolve) => {
+            resolveStop = resolve;
+          }),
+      );
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite;
+
+      const lease = await startOtlpReceiverIfNeeded(testSuite);
+      const stopping = stopOtlpReceiverIfNeeded(lease);
+      const restarting = startOtlpReceiverIfNeeded(testSuite);
+
+      await vi.waitFor(() => expect(mockStopOTLPReceiver).toHaveBeenCalledTimes(1));
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1);
+
+      resolveStop();
+      await Promise.all([stopping, restarting]);
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(2);
+    });
+
+    it('should preserve a receiver lease when an overlapping shutdown fails', async () => {
+      let rejectStop!: (error: Error) => void;
+      mockStopOTLPReceiver.mockImplementationOnce(
+        () =>
+          new Promise<void>((_resolve, reject) => {
+            rejectStop = reject;
+          }),
+      );
+      const testSuite = {
+        providers: [],
+        prompts: [],
+        tracing: {
+          enabled: true,
+          otlp: { http: { enabled: true, port: 4318, host: '127.0.0.1' } },
+        },
+      } as unknown as TestSuite;
+
+      const lease = await startOtlpReceiverIfNeeded(testSuite);
+      const stopping = stopOtlpReceiverIfNeeded(lease);
+      const restarting = startOtlpReceiverIfNeeded(testSuite);
+
+      await vi.waitFor(() => expect(mockStopOTLPReceiver).toHaveBeenCalledTimes(1));
+      rejectStop(new Error('shutdown failed'));
+
+      await expect(stopping).resolves.toBeUndefined();
+      await expect(restarting).resolves.toBe(true);
+      expect(mockStartOTLPReceiver).toHaveBeenCalledTimes(1);
+      expect(isOtlpReceiverStarted()).toBe(true);
+
+      await stopOtlpReceiverIfNeeded(true);
+      expect(mockStopOTLPReceiver).toHaveBeenCalledTimes(2);
+      expect(isOtlpReceiverStarted()).toBe(false);
+    });
+  });
+});