promptfoo--promptfoo/examples/integration-opentelemetry/javascript/provider-simple-traced.js

// provider-simple-traced.js
// RAG/Agent provider with intricate OpenTelemetry tracing

const { trace, context, SpanStatusCode } = require('@opentelemetry/api');
const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); // Use BatchSpanProcessor
const { resourceFromAttributes } = require('@opentelemetry/resources');
const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } = require('@opentelemetry/semantic-conventions');

// Configure OTLP exporter
const exporterUrl = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces';
console.log('[Provider] Configuring OTLP exporter with URL:', exporterUrl);
const exporter = new OTLPTraceExporter({
  url: exporterUrl,
});

// Use BatchSpanProcessor for better timing handling
const spanProcessor = new BatchSpanProcessor(exporter, {
  maxQueueSize: 100,
  maxExportBatchSize: 10,
  scheduledDelayMillis: 500, // Wait 500ms before exporting
  exportTimeoutMillis: 30000,
});

// Initialize OpenTelemetry with span processor in constructor (v2.x API)
const provider = new NodeTracerProvider({
  resource: resourceFromAttributes({
    [ATTR_SERVICE_NAME]: 'simple-traced-provider',
    [ATTR_SERVICE_VERSION]: '1.0.0',
  }),
  spanProcessors: [spanProcessor],
});
provider.register();

// Get a tracer
const tracer = trace.getTracer('simple-traced-provider', '1.0.0');

// Fixed helper function that properly manages span lifecycle
async function runInSpan(spanOrName, attributesOrFn, maybeFn) {
  let span;
  let fn;
  let attributes = {};

  // Handle overloaded parameters
  if (typeof spanOrName === 'string') {
    // Called with (name, attributes, fn) or (name, fn)
    if (typeof attributesOrFn === 'function') {
      fn = attributesOrFn;
    } else {
      attributes = attributesOrFn || {};
      fn = maybeFn;
    }
    span = tracer.startSpan(spanOrName, { attributes });
  } else {
    // Called with (span, fn) - original pattern
    span = spanOrName;
    fn = attributesOrFn;
  }

  const ctx = trace.setSpan(context.active(), span);

  try {
    const result = await context.with(ctx, fn);
    span.setStatus({ code: SpanStatusCode.OK });
    return result;
  } catch (error) {
    span.recordException(error);
    span.setStatus({
      code: SpanStatusCode.ERROR,
      message: error.message,
    });
    throw error;
  } finally {
    span.end();
  }
}

// Provider implementation
class SimpleTracedProvider {
  id() {
    return 'simple-traced-provider';
  }

  async callApi(prompt, promptfooContext) {
    console.log('[Provider] Called with:', {
      traceparent: promptfooContext?.traceparent,
      evaluationId: promptfooContext?.evaluationId,
      testCaseId: promptfooContext?.testCaseId,
    });

    // Check if we have trace context from Promptfoo
    if (promptfooContext?.traceparent) {
      // Parse W3C trace context
      const matches = promptfooContext.traceparent.match(
        /^(\d{2})-([a-f0-9]{32})-([a-f0-9]{16})-(\d{2})$/,
      );

      if (matches) {
        const [, _version, traceId, parentId, traceFlags] = matches;
        console.log('[Provider] Using trace context:', { traceId, parentId });

        // Create parent context from Promptfoo's trace
        const parentCtx = trace.setSpanContext(context.active(), {
          traceId,
          spanId: parentId,
          traceFlags: Number.parseInt(traceFlags, 16),
          isRemote: true,
        });

        // Run our operations within the parent context
        return context.with(parentCtx, () => this._tracedCallApi(prompt, promptfooContext));
      }
    }

    console.log('[Provider] No trace context, running without tracing');
    return this._untracedCallApi(prompt, promptfooContext);
  }

  async _tracedCallApi(prompt, promptfooContext) {
    // Use the improved runInSpan for the main workflow
    return runInSpan(
      'rag_agent_workflow',
      {
        'promptfoo.evaluation_id': promptfooContext.evaluationId,
        'promptfoo.test_case_id': promptfooContext.testCaseId,
        'prompt.text': prompt,
        'prompt.length': prompt.length,
        'agent.type': 'rag_assistant',
        'agent.version': '2.0',
      },
      async () => {
        const span = trace.getSpan(context.active());
        const startTime = Date.now();
        let totalTokens = 0;
        let userIntent;
        const documents = [];

        // Step 1: Query Analysis
        await runInSpan(
          'query_analysis',
          {
            'step.type': 'preprocessing',
            'model.name': 'gpt-3.5-turbo',
          },
          async () => {
            const span = trace.getSpan(context.active());
            span.addEvent('analyzing_user_intent');
            const analysisDelay = 250 + Math.random() * 100;
            await new Promise((resolve) => setTimeout(resolve, analysisDelay));

            userIntent = {
              type: prompt.toLowerCase().includes('compare')
                ? 'comparison'
                : prompt.toLowerCase().includes('explain')
                  ? 'explanation'
                  : 'general',
              entities: ['quantum computing', 'classical computing'],
              complexity: 'medium',
            };

            span.setAttributes({
              'intent.type': userIntent.type,
              'intent.entities': JSON.stringify(userIntent.entities),
              'intent.complexity': userIntent.complexity,
              'tokens.used': 120,
            });
            totalTokens += 120;
          },
        );

        // Step 2: Document Retrieval
        await runInSpan(
          'document_retrieval',
          {
            'retrieval.method': 'vector_similarity',
            'retrieval.index': 'technical_docs',
          },
          async () => {
            const retrievalSpan = trace.getSpan(context.active());

            // Simulate multiple retrieval attempts
            for (let i = 0; i < 3; i++) {
              await runInSpan(
                `retrieve_document_${i}`,
                {
                  'document.index': i,
                  'search.query': userIntent.entities.join(' '),
                  'tool.name': 'search_corpus',
                  'tool.arguments': JSON.stringify({
                    query: userIntent.entities.join(' '),
                    document_index: i,
                  }),
                },
                async () => {
                  const docSpan = trace.getSpan(context.active());
                  const retrievalDelay = 150 + i * 50 + Math.random() * 50;
                  await new Promise((resolve) => setTimeout(resolve, retrievalDelay));

                  const doc = {
                    id: `doc_${i + 1}`,
                    title: `Technical Document ${i + 1}`,
                    relevance_score: 0.95 - i * 0.1,
                    chunk_count: 5,
                    source: i === 0 ? 'arxiv' : i === 1 ? 'wikipedia' : 'textbook',
                  };

                  docSpan.setAttributes({
                    'document.id': doc.id,
                    'document.title': doc.title,
                    'document.relevance_score': doc.relevance_score,
                    'document.source': doc.source,
                  });

                  docSpan.addEvent('document_retrieved', {
                    chunk_count: doc.chunk_count,
                    processing_time_ms: retrievalDelay,
                  });

                  documents.push(doc);
                },
              );
            }

            retrievalSpan.setAttributes({
              'retrieval.document_count': documents.length,
              'retrieval.top_score': Math.max(...documents.map((d) => d.relevance_score)),
            });
          },
        );

        // Step 3: Context Augmentation
        await runInSpan(
          'context_augmentation',
          {
            'augmentation.strategy': 'rerank_and_merge',
          },
          async () => {
            const span = trace.getSpan(context.active());
            const augmentationDelay = 180 + Math.random() * 70;
            await new Promise((resolve) => setTimeout(resolve, augmentationDelay));

            span.addEvent('reranking_documents', {
              original_count: documents.length,
              strategy: 'cross_encoder',
            });

            span.addEvent('merging_contexts', {
              merge_strategy: 'weighted_concatenation',
              max_context_length: 4096,
            });

            span.setAttributes({
              'context.final_length': 3500,
              'context.document_count': 2,
              'tokens.used': 250,
            });
            totalTokens += 250;
          },
        );

        // Step 4: Reasoning Chain
        await runInSpan(
          'reasoning_chain',
          {
            'reasoning.type': 'chain_of_thought',
            'model.name': 'gpt-4',
          },
          async () => {
            const reasoningSpan = trace.getSpan(context.active());

            // Simulate multiple reasoning steps
            const reasoningSteps = [
              { step: 'identify_key_concepts', duration: 320, tokens: 180 },
              { step: 'analyze_relationships', duration: 450, tokens: 220 },
              { step: 'synthesize_answer', duration: 580, tokens: 350 },
            ];

            for (const step of reasoningSteps) {
              await runInSpan(`reasoning_${step.step}`, async () => {
                const stepSpan = trace.getSpan(context.active());
                await new Promise((resolve) => setTimeout(resolve, step.duration));

                stepSpan.addEvent(`${step.step}_completed`, {
                  processing_time_ms: step.duration,
                  confidence_score: 0.85 + Math.random() * 0.1,
                });

                stepSpan.setAttributes({
                  'step.name': step.step,
                  'step.duration_ms': step.duration,
                  'step.tokens': step.tokens,
                });

                totalTokens += step.tokens;
              });
            }

            reasoningSpan.setAttributes({
              'reasoning.total_steps': reasoningSteps.length,
              'reasoning.total_tokens': reasoningSteps.reduce((sum, s) => sum + s.tokens, 0),
            });
          },
        );

        // Step 5: Response Generation
        let response;
        await runInSpan(
          'response_generation',
          {
            'generation.type': 'augmented_response',
            'model.name': 'gpt-4',
            'tool.name': 'compose_answer',
            'tool.arguments': JSON.stringify({
              citation_count: documents.length,
              tone: 'explanatory',
            }),
          },
          async () => {
            const span = trace.getSpan(context.active());
            const generationDelay = 750 + Math.random() * 200;
            await new Promise((resolve) => setTimeout(resolve, generationDelay));

            response = {
              text:
                `Based on my analysis of ${documents.length} technical documents, here's a comprehensive explanation:\n\n` +
                `${userIntent.entities.join(' and ')} are fascinating topics in computer science. ` +
                `After analyzing multiple sources including arxiv papers and textbooks, I can provide the following insights:\n\n` +
                `1. Core Concepts: The fundamental principles involve...\n` +
                `2. Key Differences: When comparing these technologies...\n` +
                `3. Practical Applications: In real-world scenarios...\n\n` +
                `This synthesis is based on recent research and established knowledge in the field.`,
              citations: documents.map((d) => ({
                id: d.id,
                title: d.title,
                relevance: d.relevance_score,
              })),
              confidence: 0.92,
            };

            span.setAttributes({
              'response.length': response.text.length,
              'response.citations_count': response.citations.length,
              'response.confidence': response.confidence,
              'tokens.used': 450,
            });

            span.addEvent('response_finalized', {
              word_count: response.text.split(' ').length,
              paragraph_count: response.text.split('\n\n').length,
            });

            totalTokens += 450;
          },
        );

        // Add final span attributes
        span.setAttributes({
          'workflow.total_duration_ms': Date.now() - startTime,
          'workflow.total_steps': 5,
          'workflow.total_tokens': totalTokens,
          'workflow.success': true,
          'response.confidence': response.confidence,
        });

        span.addEvent('workflow_completed', {
          total_processing_time_ms: Date.now() - startTime,
          documents_used: documents.length,
          reasoning_steps: 3,
        });

        // Force flush to ensure spans are sent
        try {
          console.log('[Provider] Flushing spans...');
          await spanProcessor.forceFlush();
          console.log('[Provider] Spans exported successfully');
        } catch (error) {
          console.error('[Provider] Failed to flush spans:', error.message);
        }

        return {
          output: response.text,
          tokenUsage: {
            total: totalTokens,
            prompt: Math.floor(totalTokens * 0.4),
            completion: Math.floor(totalTokens * 0.6),
          },
          metadata: {
            citations: response.citations,
            confidence: response.confidence,
            workflow_duration_ms: Date.now() - startTime,
          },
        };
      },
    );
  }

  async _untracedCallApi(prompt, promptfooContext) {
    // Simple implementation without tracing
    await new Promise((resolve) => setTimeout(resolve, 100));

    const topic = prompt.toLowerCase().includes('quantum')
      ? 'quantum computing'
      : 'machine learning';
    return {
      output: `Here's a simple explanation of ${topic}: It's a fascinating field that involves...`,
      tokenUsage: {
        total: 50,
        prompt: 30,
        completion: 20,
      },
    };
  }
}

module.exports = SimpleTracedProvider;