chore: import upstream snapshot with attribution

examples/integration-opentelemetry/built-in/README.md

@@ -0,0 +1,124 @@
+# integration-opentelemetry/built-in (OpenTelemetry Built-in Tracing)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example integration-opentelemetry/built-in
+cd integration-opentelemetry/built-in
+```
+
+This example demonstrates promptfoo's built-in OpenTelemetry tracing for LLM provider calls.
+
+## Quick Start
+
+1. **Set up environment variables:**
+
+```bash
+# Required for the providers you want to test
+export OPENAI_API_KEY=sk-...
+export ANTHROPIC_API_KEY=sk-ant-...
+# Add other provider keys as needed
+```
+
+2. **Run the evaluation:**
+
+```bash
+npx promptfoo eval -c promptfooconfig.yaml
+```
+
+3. **View traces in the UI:**
+
+```bash
+npx promptfoo view
+```
+
+Navigate to the Traces tab to see detailed span information.
+
+## Configuration
+
+Tracing is enabled by default. Configure via environment variables:
+
+| Variable                      | Default     | Description                            |
+| ----------------------------- | ----------- | -------------------------------------- |
+| `PROMPTFOO_DISABLE_TRACING`   | `false`     | Set to `true` to disable tracing       |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | -           | Export traces to external OTLP backend |
+| `OTEL_SERVICE_NAME`           | `promptfoo` | Service name in traces                 |
+
+## Viewing Traces Externally
+
+### With Jaeger
+
+1. Start Jaeger:
+
+```bash
+docker run -d --name jaeger \
+  -e COLLECTOR_OTLP_ENABLED=true \
+  -p 16686:16686 \
+  -p 4318:4318 \
+  jaegertracing/all-in-one:latest
+```
+
+2. Run eval with OTLP export:
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 npx promptfoo eval
+```
+
+3. View at http://localhost:16686
+
+### With Honeycomb
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=https://api.honeycomb.io \
+OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=YOUR_API_KEY" \
+npx promptfoo eval
+```
+
+## Trace Attributes
+
+Each LLM call span includes:
+
+### GenAI Semantic Conventions
+
+- `gen_ai.system` - Provider system (openai, anthropic, etc.)
+- `gen_ai.operation.name` - Operation type (chat, completion, embedding)
+- `gen_ai.request.model` - Requested model name
+- `gen_ai.request.max_tokens` - Max tokens setting
+- `gen_ai.request.temperature` - Temperature setting
+- `gen_ai.usage.input_tokens` - Prompt tokens used
+- `gen_ai.usage.output_tokens` - Completion tokens used
+- `gen_ai.usage.total_tokens` - Total tokens
+- `gen_ai.usage.cached_tokens` - Cached tokens (Anthropic)
+- `gen_ai.usage.reasoning_tokens` - Reasoning tokens (o1 models)
+- `gen_ai.response.model` - Actual model used
+- `gen_ai.response.id` - Provider response ID
+- `gen_ai.response.finish_reasons` - Finish reasons
+
+### Promptfoo Attributes
+
+- `promptfoo.provider.id` - Provider identifier
+- `promptfoo.eval.id` - Evaluation run ID
+- `promptfoo.test.index` - Test case index
+- `promptfoo.prompt.label` - Prompt label
+
+## Supported Providers
+
+All major providers are instrumented:
+
+| Provider          | Tracing Support |
+| ----------------- | --------------- |
+| OpenAI            | ✓               |
+| Anthropic         | ✓               |
+| Azure OpenAI      | ✓               |
+| AWS Bedrock       | ✓               |
+| Google Vertex AI  | ✓               |
+| Ollama            | ✓               |
+| Mistral           | ✓               |
+| Cohere            | ✓               |
+| Huggingface       | ✓               |
+| IBM Watsonx       | ✓               |
+| HTTP              | ✓               |
+| OpenRouter        | ✓               |
+| Replicate         | ✓               |
+| OpenAI-compatible | ✓ (inherited)   |
+| Cloudflare AI     | ✓ (inherited)   |

examples/integration-opentelemetry/built-in/promptfooconfig.yaml

@@ -0,0 +1,58 @@
+# OpenTelemetry Tracing Example Configuration
+# This config tests multiple providers to verify tracing instrumentation
+
+description: OTEL Tracing Validation
+
+prompts:
+  - 'What is 2 + 2? Answer with just the number.'
+  - label: reasoning-prompt
+    raw: 'Explain why the sky is blue in one sentence.'
+
+providers:
+  # OpenAI (Category A - Direct instrumentation)
+  - id: openai:gpt-4o-mini
+    config:
+      temperature: 0.1
+      max_tokens: 50
+
+  # Anthropic (Category A - Direct instrumentation)
+  - id: anthropic:claude-haiku-4-5-20251001
+    config:
+      temperature: 0.1
+      max_tokens: 50
+
+  # Azure OpenAI (Category A - Direct instrumentation)
+  # Uncomment if you have Azure configured
+  # - id: azure:gpt-4o-mini
+  #   config:
+  #     deployment_id: your-deployment
+  #     temperature: 0.1
+
+  # Ollama (Category A - Direct instrumentation)
+  # Uncomment if you have Ollama running locally
+  # - id: ollama:llama2
+  #   config:
+  #     temperature: 0.1
+
+  # OpenRouter (Category A - Direct instrumentation with OpenAI inheritance)
+  # Uncomment if you have OpenRouter API key
+  # - id: openrouter:openai/gpt-5.4
+  #   config:
+  #     temperature: 0.1
+
+tests:
+  - vars:
+      topic: mathematics
+    assert:
+      - type: contains
+        value: '4'
+
+  - vars:
+      topic: science
+    assert:
+      - type: llm-rubric
+        value: The response should mention light scattering or wavelengths
+
+# Evaluation settings
+evaluateOptions:
+  maxConcurrency: 2

examples/integration-opentelemetry/built-in/validate-tracing.ts

@@ -0,0 +1,209 @@
+#!/usr/bin/env npx tsx
+
+/**
+ * OTEL Tracing Validation Script
+ *
+ * This script validates that OpenTelemetry tracing is working correctly
+ * by making provider calls and verifying spans are created with the
+ * expected attributes.
+ *
+ * Usage:
+ *   npx tsx examples/integration-opentelemetry/built-in/validate-tracing.ts
+ *
+ * Prerequisites:
+ *   - Set OPENAI_API_KEY environment variable
+ *   - Or modify the providers array to use a different provider
+ */
+
+import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+
+// Set up OTEL before importing providers
+const memoryExporter = new InMemorySpanExporter();
+const tracerProvider = new NodeTracerProvider({
+  spanProcessors: [new SimpleSpanProcessor(memoryExporter)],
+});
+tracerProvider.register();
+
+// Now import providers (after OTEL is set up)
+import { OpenAiChatCompletionProvider } from '../../src/providers/openai/chat';
+import { GenAIAttributes, PromptfooAttributes } from '../../src/tracing/genaiTracer';
+
+interface ValidationResult {
+  name: string;
+  passed: boolean;
+  message: string;
+}
+
+async function validateProvider(
+  providerName: string,
+  provider: { callApi: (prompt: string, context?: any) => Promise<any> },
+  prompt: string,
+): Promise<ValidationResult[]> {
+  const results: ValidationResult[] = [];
+
+  console.log(`\n🔍 Testing ${providerName}...`);
+  memoryExporter.reset();
+
+  try {
+    // Make the API call
+    const response = await provider.callApi(prompt, {
+      test: { vars: { __testIdx: 42 } },
+      prompt: { label: 'validation-test' },
+    });
+
+    // Check response
+    if (response.error) {
+      results.push({
+        name: `${providerName}: API Call`,
+        passed: false,
+        message: `API error: ${response.error}`,
+      });
+      return results;
+    }
+
+    results.push({
+      name: `${providerName}: API Call`,
+      passed: true,
+      message: `Got response: ${response.output?.substring(0, 50)}...`,
+    });
+
+    // Wait for spans to be processed
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    // Get exported spans
+    const spans = memoryExporter.getFinishedSpans();
+
+    // Validate span was created
+    if (spans.length === 0) {
+      results.push({
+        name: `${providerName}: Span Creation`,
+        passed: false,
+        message: 'No spans were created',
+      });
+      return results;
+    }
+
+    results.push({
+      name: `${providerName}: Span Creation`,
+      passed: true,
+      message: `Created ${spans.length} span(s)`,
+    });
+
+    const span = spans[0];
+
+    // Validate GenAI attributes
+    const system = span.attributes[GenAIAttributes.SYSTEM];
+    results.push({
+      name: `${providerName}: gen_ai.system`,
+      passed: !!system,
+      message: system ? `Value: ${system}` : 'Missing attribute',
+    });
+
+    const opName = span.attributes[GenAIAttributes.OPERATION_NAME];
+    results.push({
+      name: `${providerName}: gen_ai.operation.name`,
+      passed: !!opName,
+      message: opName ? `Value: ${opName}` : 'Missing attribute',
+    });
+
+    const model = span.attributes[GenAIAttributes.REQUEST_MODEL];
+    results.push({
+      name: `${providerName}: gen_ai.request.model`,
+      passed: !!model,
+      message: model ? `Value: ${model}` : 'Missing attribute',
+    });
+
+    // Validate token usage (if response has it)
+    if (response.tokenUsage) {
+      const inputTokens = span.attributes[GenAIAttributes.USAGE_INPUT_TOKENS];
+      const outputTokens = span.attributes[GenAIAttributes.USAGE_OUTPUT_TOKENS];
+
+      results.push({
+        name: `${providerName}: Token Usage`,
+        passed: inputTokens !== undefined && outputTokens !== undefined,
+        message: `Input: ${inputTokens}, Output: ${outputTokens}`,
+      });
+    }
+
+    // Validate Promptfoo attributes
+    const providerId = span.attributes[PromptfooAttributes.PROVIDER_ID];
+    results.push({
+      name: `${providerName}: promptfoo.provider.id`,
+      passed: !!providerId,
+      message: providerId ? `Value: ${providerId}` : 'Missing attribute',
+    });
+
+    // Validate span status
+    results.push({
+      name: `${providerName}: Span Status`,
+      passed: span.status.code === 1, // SpanStatusCode.OK
+      message: span.status.code === 1 ? 'OK' : `Error: ${span.status.message}`,
+    });
+  } catch (error) {
+    results.push({
+      name: `${providerName}: Execution`,
+      passed: false,
+      message: `Exception: ${error instanceof Error ? error.message : String(error)}`,
+    });
+  }
+
+  return results;
+}
+
+async function main() {
+  console.log('═══════════════════════════════════════════════════════════════');
+  console.log('  OTEL Tracing Validation Script');
+  console.log('═══════════════════════════════════════════════════════════════');
+
+  const allResults: ValidationResult[] = [];
+  const prompt = 'What is 2 + 2? Answer with just the number.';
+
+  // Test OpenAI if API key is available
+  if (process.env.OPENAI_API_KEY) {
+    const openaiProvider = new OpenAiChatCompletionProvider('gpt-4o-mini', {
+      config: { temperature: 0, max_tokens: 10 },
+    });
+
+    const openaiResults = await validateProvider('OpenAI', openaiProvider, prompt);
+    allResults.push(...openaiResults);
+  } else {
+    console.log('\n⚠️  OPENAI_API_KEY not set, skipping OpenAI validation');
+  }
+
+  // Add more providers here as needed
+  // if (process.env.ANTHROPIC_API_KEY) { ... }
+  // if (process.env.AZURE_OPENAI_API_KEY) { ... }
+
+  // Print summary
+  console.log('\n═══════════════════════════════════════════════════════════════');
+  console.log('  Validation Summary');
+  console.log('═══════════════════════════════════════════════════════════════\n');
+
+  let passed = 0;
+  let failed = 0;
+
+  for (const result of allResults) {
+    const icon = result.passed ? '✅' : '❌';
+    console.log(`${icon} ${result.name}`);
+    console.log(`   ${result.message}\n`);
+
+    if (result.passed) {
+      passed++;
+    } else {
+      failed++;
+    }
+  }
+
+  console.log('───────────────────────────────────────────────────────────────');
+  console.log(`  Total: ${passed + failed} | Passed: ${passed} | Failed: ${failed}`);
+  console.log('───────────────────────────────────────────────────────────────');
+
+  // Clean up
+  await tracerProvider.shutdown();
+
+  // Exit with error code if any tests failed
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+main().catch(console.error);