chore: import upstream snapshot with attribution

examples/integration-opentelemetry/javascript/README.md

@@ -0,0 +1,242 @@
+# integration-opentelemetry/javascript (OpenTelemetry Tracing Example)
+
+This example demonstrates how to use OpenTelemetry to trace the internal operations of your LLM providers during Promptfoo evaluations.
+
+## Quick Start
+
+```bash
+npx promptfoo@latest init --example integration-opentelemetry/javascript
+cd integration-opentelemetry/javascript
+npm install
+npx promptfoo@latest eval
+npx promptfoo@latest view
+```
+
+To run the trajectory assertion variant from this directory, use:
+
+```bash
+npx promptfoo@latest eval -c promptfooconfig.trajectory.yaml --no-cache
+```
+
+## Environment Variables
+
+This example requires no API keys - it uses a simulated provider that demonstrates tracing patterns.
+
+## Overview
+
+Promptfoo's OpenTelemetry integration allows you to:
+
+- Trace internal operations of your providers without a custom SDK
+- Use standard OpenTelemetry libraries in any language
+- Send traces to any OpenTelemetry-compatible backend
+- Correlate traces with specific test cases and evaluations
+
+## How It Works
+
+1. **OTLP receiver starts automatically** - Promptfoo ensures the receiver is ready before evaluations begin
+2. **Promptfoo generates a trace context** for each test case evaluation
+3. **The trace context is passed to providers** via the `traceparent` field
+4. **Providers create child spans** using standard OpenTelemetry SDKs
+5. **Traces are sent to Promptfoo's OTLP endpoint** (port 4318 by default)
+6. **Promptfoo correlates traces** with evaluations for analysis
+
+## Files in This Example
+
+| File                        | Description                                           |
+| --------------------------- | ----------------------------------------------------- |
+| `promptfooconfig.yaml`      | Evaluation config with tracing enabled and assertions |
+| `provider-simple-traced.js` | Simulated RAG provider with comprehensive tracing     |
+| `trace-assertions.js`       | Custom JavaScript assertion for trace validation      |
+| `package.json`              | OpenTelemetry dependencies (v2.x API)                 |
+
+## Tracing Configuration
+
+Enable tracing in your `promptfooconfig.yaml`:
+
+```yaml
+tracing:
+  enabled: true
+  otlp:
+    http:
+      enabled: true
+      port: 4318
+      host: '0.0.0.0'
+```
+
+## Instrumenting Your Provider
+
+The provider receives trace context from Promptfoo via the `traceparent` field. Here's the pattern used in this example:
+
+```javascript
+const { trace, context, SpanStatusCode } = require('@opentelemetry/api');
+const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-node');
+const { resourceFromAttributes } = require('@opentelemetry/resources');
+const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
+
+// Initialize OpenTelemetry (v2.x API)
+const exporter = new OTLPTraceExporter({
+  url: 'http://localhost:4318/v1/traces',
+});
+
+const provider = new NodeTracerProvider({
+  resource: resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: 'my-provider',
+  }),
+  spanProcessors: [new BatchSpanProcessor(exporter)],
+});
+provider.register();
+
+const tracer = trace.getTracer('my-provider');
+
+module.exports = {
+  async callApi(prompt, promptfooContext) {
+    // Parse trace context from Promptfoo
+    if (promptfooContext?.traceparent) {
+      const matches = promptfooContext.traceparent.match(
+        /^(\d{2})-([a-f0-9]{32})-([a-f0-9]{16})-(\d{2})$/,
+      );
+      if (matches) {
+        const [, , traceId, parentId, traceFlags] = matches;
+
+        // Create parent context
+        const parentCtx = trace.setSpanContext(context.active(), {
+          traceId,
+          spanId: parentId,
+          traceFlags: parseInt(traceFlags, 16),
+          isRemote: true,
+        });
+
+        // Run operations within parent context
+        return context.with(parentCtx, async () => {
+          const span = tracer.startSpan('my_operation');
+          try {
+            // Your provider logic here...
+            span.setStatus({ code: SpanStatusCode.OK });
+            return { output: 'result' };
+          } catch (error) {
+            span.recordException(error);
+            span.setStatus({ code: SpanStatusCode.ERROR });
+            throw error;
+          } finally {
+            span.end();
+          }
+        });
+      }
+    }
+
+    return { output: 'result without tracing' };
+  },
+};
+```
+
+## Trace-Based Assertions
+
+This example demonstrates several trace assertion types:
+
+```yaml
+assert:
+  # Count spans matching a pattern
+  - type: trace-span-count
+    value:
+      pattern: 'retrieve_document_*'
+      min: 3
+      max: 3
+
+  # Check span duration
+  - type: trace-span-duration
+    value:
+      pattern: 'rag_agent_workflow'
+      max: 5000 # milliseconds
+
+  # Check for error spans
+  - type: trace-error-spans
+    value:
+      max_count: 0
+```
+
+The trajectory-specific config at `promptfooconfig.trajectory.yaml` adds:
+
+- `trajectory:tool-used`
+- `trajectory:tool-args-match`
+- `trajectory:tool-sequence`
+- `trajectory:step-count`
+
+Promptfoo accepts generic tool span attributes such as `tool.name` and `tool.arguments`, and it also recognizes Vercel AI SDK telemetry attributes such as `ai.toolCall.name`, `ai.toolCall.args`, `ai.toolCall.arguments`, and `ai.toolCall.input`.
+
+## Viewing Traces
+
+After running an evaluation, view traces in the web UI:
+
+```bash
+npx promptfoo@latest view
+```
+
+Click on any test result to see the "Trace Timeline" section showing:
+
+- Hierarchical span visualization
+- Duration bars showing relative timing
+- Status indicators (OK/ERROR)
+- Span attributes and events
+
+## Environment Variables
+
+Configure OpenTelemetry using standard environment variables:
+
+```bash
+# Custom endpoint (defaults to Promptfoo's receiver)
+export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
+
+# Headers for authentication with external collectors
+export OTEL_EXPORTER_OTLP_HEADERS="api-key=your-key"
+
+# Enable tracing via environment variable
+export PROMPTFOO_TRACING_ENABLED=true
+```
+
+## Forward to External Collectors
+
+Send traces to Jaeger, Honeycomb, or other OTLP-compatible backends:
+
+```yaml
+tracing:
+  enabled: true
+  forwarding:
+    enabled: true
+    endpoint: 'http://jaeger:4318'
+    headers:
+      'api-key': '${JAEGER_API_KEY}'
+```
+
+## Troubleshooting
+
+### Context Naming Conflicts
+
+If you see `context.active is not a function`, the OpenTelemetry `context` API conflicts with Promptfoo's context parameter. Rename the parameter:
+
+```javascript
+async callApi(prompt, promptfooContext) {
+  // Use promptfooContext for Promptfoo's context
+  // Use context from @opentelemetry/api for tracing
+}
+```
+
+### Traces Not Appearing
+
+1. Verify `tracing.enabled: true` in config
+2. Check OTLP receiver is running (look for port 4318 in logs)
+3. Ensure trace context is properly parsed from `promptfooContext.traceparent`
+4. Call `spanProcessor.forceFlush()` before returning from provider
+
+## Dependencies
+
+This example uses OpenTelemetry v2.x packages:
+
+| Package                                   | Version  | Purpose                  |
+| ----------------------------------------- | -------- | ------------------------ |
+| `@opentelemetry/api`                      | ^1.9.0   | Core tracing API         |
+| `@opentelemetry/sdk-trace-node`           | ^2.0.0   | Node.js tracer provider  |
+| `@opentelemetry/exporter-trace-otlp-http` | ^0.200.0 | OTLP HTTP exporter       |
+| `@opentelemetry/resources`                | ^2.0.0   | Resource attributes      |
+| `@opentelemetry/semantic-conventions`     | ^1.28.0  | Standard attribute names |

examples/integration-opentelemetry/javascript/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@promptfoo/opentelemetry-tracing-example",
+  "version": "1.0.0",
+  "license": "MIT",
+  "private": true,
+  "description": "OpenTelemetry tracing integration with Promptfoo evaluations",
+  "scripts": {
+    "eval": "promptfoo eval",
+    "view": "promptfoo view"
+  },
+  "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/sdk-trace-node": "^2.5.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.219.0",
+    "@opentelemetry/resources": "^2.5.0",
+    "@opentelemetry/semantic-conventions": "^1.39.0"
+  }
+}

examples/integration-opentelemetry/javascript/promptfooconfig.trajectory.yaml

@@ -0,0 +1,63 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Trajectory assertions with OpenTelemetry traces
+
+# Promptfoo accepts generic tool span attributes like tool.name/tool.arguments
+# and Vercel AI SDK telemetry attributes like ai.toolCall.name/ai.toolCall.args.
+
+prompts:
+  - 'Explain how {{topic}} works in simple terms'
+
+providers:
+  - file://provider-simple-traced.js
+
+tests:
+  - vars:
+      topic: 'quantum computing'
+    metadata:
+      tracingEnabled: true
+      testCaseId: 'trajectory-test-case-1'
+    assert:
+      - type: trajectory:tool-used
+        value:
+          pattern: 'search_*'
+          min: 3
+          max: 3
+
+      - type: trajectory:tool-used
+        value: compose_answer
+
+      - type: trajectory:tool-args-match
+        value:
+          name: search_corpus
+          args:
+            query: quantum computing classical computing
+
+      - type: trajectory:tool-sequence
+        value:
+          mode: exact
+          steps:
+            - search_corpus
+            - search_corpus
+            - search_corpus
+            - compose_answer
+
+      - type: trajectory:step-count
+        value:
+          type: reasoning
+          min: 4
+          max: 4
+
+      - type: trajectory:step-count
+        value:
+          pattern: 'retrieve_document_*'
+          min: 3
+          max: 3
+
+tracing:
+  enabled: true
+  otlp:
+    http:
+      enabled: true
+      port: 4318
+      host: '0.0.0.0'
+      acceptFormats: ['json']

examples/integration-opentelemetry/javascript/promptfooconfig.yaml

@@ -0,0 +1,132 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: OpenTelemetry tracing with trace-based assertions
+
+providers:
+  - file://provider-simple-traced.js
+
+prompts:
+  - 'Explain how {{topic}} works in simple terms'
+
+tests:
+  - vars:
+      topic: 'quantum computing'
+    metadata:
+      tracingEnabled: true
+      testCaseId: 'test-case-1'
+    assert:
+      # Original javascript assertion
+      - type: javascript
+        value: file://trace-assertions.js
+
+      # Ensure all expected spans are present
+      - type: trace-span-count
+        value:
+          pattern: 'rag_agent_workflow'
+          min: 1
+          max: 1
+
+      # Ensure we retrieve exactly 3 documents
+      - type: trace-span-count
+        value:
+          pattern: 'retrieve_document_*'
+          min: 3
+          max: 3
+
+      # Ensure all reasoning steps occur
+      - type: trace-span-count
+        value:
+          pattern: 'reasoning_*'
+          min: 3
+
+      # Ensure the overall workflow completes quickly
+      - type: trace-span-duration
+        value:
+          pattern: 'rag_agent_workflow'
+          max: 5000 # 5 seconds max
+
+      # Ensure individual operations don't take too long
+      - type: trace-span-duration
+        value:
+          pattern: '*'
+          max: 1000 # No single span should exceed 1 second
+
+      # Ensure no errors occur
+      - type: trace-error-spans
+        value:
+          max_count: 0
+
+  - vars:
+      topic: 'machine learning'
+    metadata:
+      tracingEnabled: true
+      testCaseId: 'test-case-2'
+    assert:
+      # Original javascript assertion
+      - type: javascript
+        value: file://trace-assertions.js
+
+      # Same trace assertions as above
+      - type: trace-span-count
+        value:
+          pattern: 'rag_agent_workflow'
+          min: 1
+          max: 1
+
+      - type: trace-span-count
+        value:
+          pattern: 'retrieve_document_*'
+          min: 3
+          max: 3
+
+      - type: trace-span-count
+        value:
+          pattern: 'reasoning_*'
+          min: 3
+
+      - type: trace-span-duration
+        value:
+          pattern: 'rag_agent_workflow'
+          max: 5000
+
+      - type: trace-span-duration
+        value:
+          pattern: '*'
+          max: 1000
+
+      - type: trace-error-spans
+        value:
+          max_count: 0
+
+# Default assertions that apply to all test cases
+defaultTest:
+  assert:
+    # Monitor 95th percentile latency (metric only, won't fail tests)
+    - type: trace-span-duration
+      value:
+        pattern: '*'
+        max: 2000
+        percentile: 95
+      weight: 0 # This makes it a metric-only assertion
+      metric: p95_latency
+
+    # Ensure retrieval operations are fast
+    - type: trace-span-duration
+      value:
+        pattern: 'retrieve_document_*'
+        max: 300 # Each document retrieval should be under 300ms
+
+    # Allow up to 5% error rate (more forgiving for production)
+    - type: trace-error-spans
+      value:
+        max_percentage: 5
+        pattern: '*'
+      metric: error_rate
+
+# Tracing configuration
+tracing:
+  enabled: true
+  otlp:
+    http:
+      enabled: true
+      port: 4318
+      acceptFormats: ['json']

examples/integration-opentelemetry/javascript/provider-simple-traced.js

@@ -0,0 +1,416 @@
+// provider-simple-traced.js
+// RAG/Agent provider with intricate OpenTelemetry tracing
+
+const { trace, context, SpanStatusCode } = require('@opentelemetry/api');
+const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); // Use BatchSpanProcessor
+const { resourceFromAttributes } = require('@opentelemetry/resources');
+const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } = require('@opentelemetry/semantic-conventions');
+
+// Configure OTLP exporter
+const exporterUrl = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces';
+console.log('[Provider] Configuring OTLP exporter with URL:', exporterUrl);
+const exporter = new OTLPTraceExporter({
+  url: exporterUrl,
+});
+
+// Use BatchSpanProcessor for better timing handling
+const spanProcessor = new BatchSpanProcessor(exporter, {
+  maxQueueSize: 100,
+  maxExportBatchSize: 10,
+  scheduledDelayMillis: 500, // Wait 500ms before exporting
+  exportTimeoutMillis: 30000,
+});
+
+// Initialize OpenTelemetry with span processor in constructor (v2.x API)
+const provider = new NodeTracerProvider({
+  resource: resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: 'simple-traced-provider',
+    [ATTR_SERVICE_VERSION]: '1.0.0',
+  }),
+  spanProcessors: [spanProcessor],
+});
+provider.register();
+
+// Get a tracer
+const tracer = trace.getTracer('simple-traced-provider', '1.0.0');
+
+// Fixed helper function that properly manages span lifecycle
+async function runInSpan(spanOrName, attributesOrFn, maybeFn) {
+  let span;
+  let fn;
+  let attributes = {};
+
+  // Handle overloaded parameters
+  if (typeof spanOrName === 'string') {
+    // Called with (name, attributes, fn) or (name, fn)
+    if (typeof attributesOrFn === 'function') {
+      fn = attributesOrFn;
+    } else {
+      attributes = attributesOrFn || {};
+      fn = maybeFn;
+    }
+    span = tracer.startSpan(spanOrName, { attributes });
+  } else {
+    // Called with (span, fn) - original pattern
+    span = spanOrName;
+    fn = attributesOrFn;
+  }
+
+  const ctx = trace.setSpan(context.active(), span);
+
+  try {
+    const result = await context.with(ctx, fn);
+    span.setStatus({ code: SpanStatusCode.OK });
+    return result;
+  } catch (error) {
+    span.recordException(error);
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: error.message,
+    });
+    throw error;
+  } finally {
+    span.end();
+  }
+}
+
+// Provider implementation
+class SimpleTracedProvider {
+  id() {
+    return 'simple-traced-provider';
+  }
+
+  async callApi(prompt, promptfooContext) {
+    console.log('[Provider] Called with:', {
+      traceparent: promptfooContext?.traceparent,
+      evaluationId: promptfooContext?.evaluationId,
+      testCaseId: promptfooContext?.testCaseId,
+    });
+
+    // Check if we have trace context from Promptfoo
+    if (promptfooContext?.traceparent) {
+      // Parse W3C trace context
+      const matches = promptfooContext.traceparent.match(
+        /^(\d{2})-([a-f0-9]{32})-([a-f0-9]{16})-(\d{2})$/,
+      );
+
+      if (matches) {
+        const [, _version, traceId, parentId, traceFlags] = matches;
+        console.log('[Provider] Using trace context:', { traceId, parentId });
+
+        // Create parent context from Promptfoo's trace
+        const parentCtx = trace.setSpanContext(context.active(), {
+          traceId,
+          spanId: parentId,
+          traceFlags: Number.parseInt(traceFlags, 16),
+          isRemote: true,
+        });
+
+        // Run our operations within the parent context
+        return context.with(parentCtx, () => this._tracedCallApi(prompt, promptfooContext));
+      }
+    }
+
+    console.log('[Provider] No trace context, running without tracing');
+    return this._untracedCallApi(prompt, promptfooContext);
+  }
+
+  async _tracedCallApi(prompt, promptfooContext) {
+    // Use the improved runInSpan for the main workflow
+    return runInSpan(
+      'rag_agent_workflow',
+      {
+        'promptfoo.evaluation_id': promptfooContext.evaluationId,
+        'promptfoo.test_case_id': promptfooContext.testCaseId,
+        'prompt.text': prompt,
+        'prompt.length': prompt.length,
+        'agent.type': 'rag_assistant',
+        'agent.version': '2.0',
+      },
+      async () => {
+        const span = trace.getSpan(context.active());
+        const startTime = Date.now();
+        let totalTokens = 0;
+        let userIntent;
+        const documents = [];
+
+        // Step 1: Query Analysis
+        await runInSpan(
+          'query_analysis',
+          {
+            'step.type': 'preprocessing',
+            'model.name': 'gpt-3.5-turbo',
+          },
+          async () => {
+            const span = trace.getSpan(context.active());
+            span.addEvent('analyzing_user_intent');
+            const analysisDelay = 250 + Math.random() * 100;
+            await new Promise((resolve) => setTimeout(resolve, analysisDelay));
+
+            userIntent = {
+              type: prompt.toLowerCase().includes('compare')
+                ? 'comparison'
+                : prompt.toLowerCase().includes('explain')
+                  ? 'explanation'
+                  : 'general',
+              entities: ['quantum computing', 'classical computing'],
+              complexity: 'medium',
+            };
+
+            span.setAttributes({
+              'intent.type': userIntent.type,
+              'intent.entities': JSON.stringify(userIntent.entities),
+              'intent.complexity': userIntent.complexity,
+              'tokens.used': 120,
+            });
+            totalTokens += 120;
+          },
+        );
+
+        // Step 2: Document Retrieval
+        await runInSpan(
+          'document_retrieval',
+          {
+            'retrieval.method': 'vector_similarity',
+            'retrieval.index': 'technical_docs',
+          },
+          async () => {
+            const retrievalSpan = trace.getSpan(context.active());
+
+            // Simulate multiple retrieval attempts
+            for (let i = 0; i < 3; i++) {
+              await runInSpan(
+                `retrieve_document_${i}`,
+                {
+                  'document.index': i,
+                  'search.query': userIntent.entities.join(' '),
+                  'tool.name': 'search_corpus',
+                  'tool.arguments': JSON.stringify({
+                    query: userIntent.entities.join(' '),
+                    document_index: i,
+                  }),
+                },
+                async () => {
+                  const docSpan = trace.getSpan(context.active());
+                  const retrievalDelay = 150 + i * 50 + Math.random() * 50;
+                  await new Promise((resolve) => setTimeout(resolve, retrievalDelay));
+
+                  const doc = {
+                    id: `doc_${i + 1}`,
+                    title: `Technical Document ${i + 1}`,
+                    relevance_score: 0.95 - i * 0.1,
+                    chunk_count: 5,
+                    source: i === 0 ? 'arxiv' : i === 1 ? 'wikipedia' : 'textbook',
+                  };
+
+                  docSpan.setAttributes({
+                    'document.id': doc.id,
+                    'document.title': doc.title,
+                    'document.relevance_score': doc.relevance_score,
+                    'document.source': doc.source,
+                  });
+
+                  docSpan.addEvent('document_retrieved', {
+                    chunk_count: doc.chunk_count,
+                    processing_time_ms: retrievalDelay,
+                  });
+
+                  documents.push(doc);
+                },
+              );
+            }
+
+            retrievalSpan.setAttributes({
+              'retrieval.document_count': documents.length,
+              'retrieval.top_score': Math.max(...documents.map((d) => d.relevance_score)),
+            });
+          },
+        );
+
+        // Step 3: Context Augmentation
+        await runInSpan(
+          'context_augmentation',
+          {
+            'augmentation.strategy': 'rerank_and_merge',
+          },
+          async () => {
+            const span = trace.getSpan(context.active());
+            const augmentationDelay = 180 + Math.random() * 70;
+            await new Promise((resolve) => setTimeout(resolve, augmentationDelay));
+
+            span.addEvent('reranking_documents', {
+              original_count: documents.length,
+              strategy: 'cross_encoder',
+            });
+
+            span.addEvent('merging_contexts', {
+              merge_strategy: 'weighted_concatenation',
+              max_context_length: 4096,
+            });
+
+            span.setAttributes({
+              'context.final_length': 3500,
+              'context.document_count': 2,
+              'tokens.used': 250,
+            });
+            totalTokens += 250;
+          },
+        );
+
+        // Step 4: Reasoning Chain
+        await runInSpan(
+          'reasoning_chain',
+          {
+            'reasoning.type': 'chain_of_thought',
+            'model.name': 'gpt-4',
+          },
+          async () => {
+            const reasoningSpan = trace.getSpan(context.active());
+
+            // Simulate multiple reasoning steps
+            const reasoningSteps = [
+              { step: 'identify_key_concepts', duration: 320, tokens: 180 },
+              { step: 'analyze_relationships', duration: 450, tokens: 220 },
+              { step: 'synthesize_answer', duration: 580, tokens: 350 },
+            ];
+
+            for (const step of reasoningSteps) {
+              await runInSpan(`reasoning_${step.step}`, async () => {
+                const stepSpan = trace.getSpan(context.active());
+                await new Promise((resolve) => setTimeout(resolve, step.duration));
+
+                stepSpan.addEvent(`${step.step}_completed`, {
+                  processing_time_ms: step.duration,
+                  confidence_score: 0.85 + Math.random() * 0.1,
+                });
+
+                stepSpan.setAttributes({
+                  'step.name': step.step,
+                  'step.duration_ms': step.duration,
+                  'step.tokens': step.tokens,
+                });
+
+                totalTokens += step.tokens;
+              });
+            }
+
+            reasoningSpan.setAttributes({
+              'reasoning.total_steps': reasoningSteps.length,
+              'reasoning.total_tokens': reasoningSteps.reduce((sum, s) => sum + s.tokens, 0),
+            });
+          },
+        );
+
+        // Step 5: Response Generation
+        let response;
+        await runInSpan(
+          'response_generation',
+          {
+            'generation.type': 'augmented_response',
+            'model.name': 'gpt-4',
+            'tool.name': 'compose_answer',
+            'tool.arguments': JSON.stringify({
+              citation_count: documents.length,
+              tone: 'explanatory',
+            }),
+          },
+          async () => {
+            const span = trace.getSpan(context.active());
+            const generationDelay = 750 + Math.random() * 200;
+            await new Promise((resolve) => setTimeout(resolve, generationDelay));
+
+            response = {
+              text:
+                `Based on my analysis of ${documents.length} technical documents, here's a comprehensive explanation:\n\n` +
+                `${userIntent.entities.join(' and ')} are fascinating topics in computer science. ` +
+                `After analyzing multiple sources including arxiv papers and textbooks, I can provide the following insights:\n\n` +
+                `1. Core Concepts: The fundamental principles involve...\n` +
+                `2. Key Differences: When comparing these technologies...\n` +
+                `3. Practical Applications: In real-world scenarios...\n\n` +
+                `This synthesis is based on recent research and established knowledge in the field.`,
+              citations: documents.map((d) => ({
+                id: d.id,
+                title: d.title,
+                relevance: d.relevance_score,
+              })),
+              confidence: 0.92,
+            };
+
+            span.setAttributes({
+              'response.length': response.text.length,
+              'response.citations_count': response.citations.length,
+              'response.confidence': response.confidence,
+              'tokens.used': 450,
+            });
+
+            span.addEvent('response_finalized', {
+              word_count: response.text.split(' ').length,
+              paragraph_count: response.text.split('\n\n').length,
+            });
+
+            totalTokens += 450;
+          },
+        );
+
+        // Add final span attributes
+        span.setAttributes({
+          'workflow.total_duration_ms': Date.now() - startTime,
+          'workflow.total_steps': 5,
+          'workflow.total_tokens': totalTokens,
+          'workflow.success': true,
+          'response.confidence': response.confidence,
+        });
+
+        span.addEvent('workflow_completed', {
+          total_processing_time_ms: Date.now() - startTime,
+          documents_used: documents.length,
+          reasoning_steps: 3,
+        });
+
+        // Force flush to ensure spans are sent
+        try {
+          console.log('[Provider] Flushing spans...');
+          await spanProcessor.forceFlush();
+          console.log('[Provider] Spans exported successfully');
+        } catch (error) {
+          console.error('[Provider] Failed to flush spans:', error.message);
+        }
+
+        return {
+          output: response.text,
+          tokenUsage: {
+            total: totalTokens,
+            prompt: Math.floor(totalTokens * 0.4),
+            completion: Math.floor(totalTokens * 0.6),
+          },
+          metadata: {
+            citations: response.citations,
+            confidence: response.confidence,
+            workflow_duration_ms: Date.now() - startTime,
+          },
+        };
+      },
+    );
+  }
+
+  async _untracedCallApi(prompt, promptfooContext) {
+    // Simple implementation without tracing
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    const topic = prompt.toLowerCase().includes('quantum')
+      ? 'quantum computing'
+      : 'machine learning';
+    return {
+      output: `Here's a simple explanation of ${topic}: It's a fascinating field that involves...`,
+      tokenUsage: {
+        total: 50,
+        prompt: 30,
+        completion: 20,
+      },
+    };
+  }
+}
+
+module.exports = SimpleTracedProvider;

examples/integration-opentelemetry/javascript/trace-assertions.js

@@ -0,0 +1,103 @@
+module.exports = (output, context) => {
+  // Check if trace data is available
+  if (!context.trace) {
+    // Tracing not enabled, skip trace-based checks
+    return true;
+  }
+
+  const { spans } = context.trace;
+
+  // Check for errors in any span
+  const errorSpans = spans.filter((s) => s.statusCode >= 400);
+  if (errorSpans.length > 0) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `Found ${errorSpans.length} error spans`,
+    };
+  }
+
+  // Calculate total trace duration
+  if (spans.length > 0) {
+    const duration =
+      Math.max(...spans.map((s) => s.endTime || 0)) - Math.min(...spans.map((s) => s.startTime));
+    if (duration > 5000) {
+      // 5 seconds
+      return {
+        pass: false,
+        score: 0,
+        reason: `Trace took too long: ${duration}ms`,
+      };
+    }
+  }
+
+  // Check for specific operations - look for HTTP/API calls
+  const apiCalls = spans.filter((s) => s.name.toLowerCase().includes('http'));
+  if (apiCalls.length > 10) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `Too many API calls: ${apiCalls.length}`,
+    };
+  }
+
+  // Verify all expected RAG steps are present
+  const expectedSteps = [
+    'query_analysis',
+    'document_retrieval',
+    'context_augmentation',
+    'reasoning_chain',
+    'response_generation',
+  ];
+
+  const missingSteps = expectedSteps.filter((step) => !spans.some((s) => s.name === step));
+
+  if (missingSteps.length > 0) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `Missing RAG workflow steps: ${missingSteps.join(', ')}`,
+    };
+  }
+
+  // Check for the main workflow span (optional - may be missing due to timing)
+  const workflowSpan = spans.find((s) => s.name === 'rag_agent_workflow');
+  if (workflowSpan) {
+    // Validate workflow attributes if present
+    const attrs = workflowSpan.attributes;
+    if (!attrs['workflow.success'] || attrs['workflow.success'] !== true) {
+      return {
+        pass: false,
+        score: 0,
+        reason: 'Workflow did not complete successfully',
+      };
+    }
+  }
+
+  // Check document retrieval performance
+  const retrievalSpans = spans.filter((s) => s.name.startsWith('retrieve_document_'));
+  if (retrievalSpans.length < 3) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `Expected 3 document retrievals, found ${retrievalSpans.length}`,
+    };
+  }
+
+  // Check reasoning chain has expected sub-steps
+  const reasoningSteps = spans.filter((s) => s.name.startsWith('reasoning_'));
+  if (reasoningSteps.length < 3) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `Expected at least 3 reasoning steps, found ${reasoningSteps.length}`,
+    };
+  }
+
+  // All checks passed
+  return {
+    pass: true,
+    score: 1,
+    reason: 'Trace validation successful',
+  };
+};