promptfoo--promptfoo/test/tracing/performanceBenchmark.test.ts

/**
 * Smoke tests for OTEL instrumentation.
 *
 * These tests verify that tracing works correctly under various conditions
 * (concurrency, errors, many attributes, etc.) without asserting on timing.
 * Performance benchmarking should be done with dedicated tooling, not in CI.
 */

import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { withGenAISpan } from '../../src/tracing/genaiTracer';

import type { GenAISpanContext, GenAISpanResult } from '../../src/tracing/genaiTracer';

// Mock logger to reduce noise
vi.mock('../../src/logger', () => ({
  default: {
    debug: vi.fn(),
    info: vi.fn(),
    warn: vi.fn(),
    error: vi.fn(),
  },
}));

describe('OTEL Tracing Smoke Tests', () => {
  let tracerProvider: NodeTracerProvider;
  let memoryExporter: InMemorySpanExporter;

  beforeAll(() => {
    memoryExporter = new InMemorySpanExporter();
    tracerProvider = new NodeTracerProvider({
      spanProcessors: [new SimpleSpanProcessor(memoryExporter)],
    });
    tracerProvider.register();
  });

  afterAll(async () => {
    await tracerProvider.shutdown();
  });

  beforeEach(() => {
    memoryExporter.reset();
  });

  afterEach(() => {
    vi.resetAllMocks();
  });

  const baseContext: GenAISpanContext = {
    system: 'openai',
    operationName: 'chat',
    model: 'gpt-4',
    providerId: 'openai:gpt-4',
    maxTokens: 1000,
    temperature: 0.7,
  };

  const resultExtractor = (): GenAISpanResult => ({
    tokenUsage: { prompt: 100, completion: 50, total: 150 },
    responseId: 'chatcmpl-123',
    finishReasons: ['stop'],
  });

  describe('Basic Span Creation', () => {
    it('should create spans for traced operations', async () => {
      const iterations = 10;

      for (let i = 0; i < iterations; i++) {
        await withGenAISpan(baseContext, async () => ({ output: 'test' }));
      }

      const spans = memoryExporter.getFinishedSpans();
      expect(spans.length).toBe(iterations);
    });

    it('should create spans with result extraction', async () => {
      const iterations = 10;

      for (let i = 0; i < iterations; i++) {
        await withGenAISpan(baseContext, async () => ({ output: 'test' }), resultExtractor);
      }

      const spans = memoryExporter.getFinishedSpans();
      expect(spans.length).toBe(iterations);

      // Verify token usage attributes are set
      const span = spans[0];
      expect(span.attributes['gen_ai.usage.input_tokens']).toBe(100);
      expect(span.attributes['gen_ai.usage.output_tokens']).toBe(50);
    });

    it('should handle high concurrency without errors', async () => {
      const concurrency = 100;
      const batches = 3;

      for (let batch = 0; batch < batches; batch++) {
        await Promise.all(
          Array.from({ length: concurrency }, (_, i) =>
            withGenAISpan(
              { ...baseContext, testIndex: batch * concurrency + i },
              async () => ({ output: `Response ${i}` }),
              resultExtractor,
            ),
          ),
        );
      }

      const spans = memoryExporter.getFinishedSpans();
      expect(spans.length).toBe(concurrency * batches);
    });
  });

  describe('Attribute Handling', () => {
    it('should handle contexts with all optional attributes', async () => {
      const fullContext: GenAISpanContext = {
        system: 'openai',
        operationName: 'chat',
        model: 'gpt-4-turbo-preview',
        providerId: 'openai:gpt-4-turbo-preview',
        maxTokens: 4096,
        temperature: 0.7,
        topP: 0.9,
        topK: 40,
        stopSequences: ['END', 'STOP', '###'],
        frequencyPenalty: 0.5,
        presencePenalty: 0.5,
        evalId: 'eval-benchmark-test-123',
        testIndex: 42,
        promptLabel: 'benchmark-prompt-label',
      };

      const fullResultExtractor = (): GenAISpanResult => ({
        tokenUsage: {
          prompt: 1000,
          completion: 500,
          total: 1500,
          cached: 200,
          completionDetails: {
            reasoning: 300,
            acceptedPrediction: 150,
            rejectedPrediction: 50,
          },
        },
        responseModel: 'gpt-4-turbo-preview-2024-01-01',
        responseId: 'chatcmpl-benchmark123456789',
        finishReasons: ['stop'],
      });

      const iterations = 10;

      for (let i = 0; i < iterations; i++) {
        await withGenAISpan(fullContext, async () => ({ output: 'test' }), fullResultExtractor);
      }

      const spans = memoryExporter.getFinishedSpans();
      expect(spans.length).toBe(iterations);

      // Verify key attributes are set
      const span = spans[0];
      expect(span.attributes['gen_ai.request.model']).toBe('gpt-4-turbo-preview');
      expect(span.attributes['gen_ai.request.max_tokens']).toBe(4096);
      expect(span.attributes['gen_ai.usage.input_tokens']).toBe(1000);
    });
  });

  describe('Error Handling', () => {
    it('should properly record errors in spans', async () => {
      const iterations = 10;

      for (let i = 0; i < iterations; i++) {
        try {
          await withGenAISpan(baseContext, async () => {
            throw new Error(`Test error ${i}`);
          });
        } catch {
          // Expected
        }
      }

      const spans = memoryExporter.getFinishedSpans();
      expect(spans.length).toBe(iterations);

      // Verify error is recorded
      const span = spans[0];
      expect(span.status.code).toBe(2); // SpanStatusCode.ERROR
    });
  });
});