chore: import upstream snapshot with attribution

test/integration/envPath.test.ts

@@ -0,0 +1,419 @@
+import fs from 'fs';
+import * as path from 'path';
+
+import dedent from 'dedent';
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  type MockedFunction,
+  vi,
+} from 'vitest';
+import { doEval } from '../../src/node/doEval';
+import { setupEnv } from '../../src/util/index';
+
+vi.mock('../../src/cache');
+vi.mock('../../src/evaluator');
+vi.mock('../../src/globalConfig/accounts');
+vi.mock('../../src/globalConfig/cloud', () => ({
+  cloudConfig: {
+    isEnabled: vi.fn().mockReturnValue(false),
+  },
+}));
+vi.mock('../../src/migrate');
+vi.mock('../../src/models/eval', () => {
+  const MockEval = function (this: any) {
+    this.id = 'test-eval-id';
+    this.prompts = [];
+    this.clearResults = vi.fn();
+    this.shared = false;
+    this.getTable = vi.fn().mockResolvedValue({ body: [] });
+  };
+  MockEval.create = vi.fn().mockResolvedValue({
+    id: 'test-eval-id',
+    prompts: [],
+    clearResults: vi.fn(),
+    shared: false,
+    getTable: vi.fn().mockResolvedValue({ body: [] }),
+  });
+  MockEval.latest = vi.fn().mockResolvedValue(null);
+  MockEval.findById = vi.fn().mockResolvedValue(null);
+  return { default: MockEval };
+});
+vi.mock('../../src/providers');
+vi.mock('../../src/share');
+vi.mock('../../src/table');
+vi.mock('../../src/util/cloud', () => ({
+  checkCloudPermissions: vi.fn().mockResolvedValue(undefined),
+  getOrgContext: vi.fn().mockResolvedValue(null),
+}));
+vi.mock('../../src/util', async () => {
+  const actual = await vi.importActual('../../src/util');
+  return {
+    ...(actual as any),
+    setupEnv: vi.fn(),
+  };
+});
+
+const mockSetupEnv = setupEnv as MockedFunction<typeof setupEnv>;
+const dedentYaml = dedent.withOptions({ escapeSpecialCharacters: false });
+
+describe('Integration: commandLineOptions.envPath', () => {
+  let tempDir: string;
+  let tempEnvFile: string;
+  let tempConfigFile: string;
+
+  beforeAll(() => {
+    tempDir = fs.mkdtempSync(path.join(__dirname, 'test-'));
+    tempEnvFile = path.join(tempDir, '.env.test');
+    tempConfigFile = path.join(tempDir, 'promptfooconfig.yaml');
+  });
+
+  afterAll(() => {
+    fs.rmSync(tempDir, { recursive: true });
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should load environment from config-specified envPath', async () => {
+    // Create test .env file
+    fs.writeFileSync(tempEnvFile, 'TEST_VAR=from_config_env');
+
+    // Create config with commandLineOptions.envPath
+    fs.writeFileSync(
+      tempConfigFile,
+      `
+commandLineOptions:
+  envPath: ${tempEnvFile}
+
+prompts:
+  - "Test prompt"
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      input: "test"
+`,
+    );
+
+    const cmdObj = { config: [tempConfigFile] };
+
+    try {
+      await doEval(cmdObj, {}, undefined, {});
+    } catch {
+      // Expected to fail due to mocked dependencies
+    }
+
+    // Should call setupEnv twice: once for CLI (undefined), once for config
+    expect(mockSetupEnv).toHaveBeenCalledTimes(2);
+    expect(mockSetupEnv).toHaveBeenNthCalledWith(1, undefined); // Phase 1: CLI
+    expect(mockSetupEnv).toHaveBeenNthCalledWith(2, tempEnvFile); // Phase 2: Config
+  });
+
+  it('should prioritize CLI envPath over config envPath', async () => {
+    const cliEnvFile = path.join(tempDir, '.env.cli');
+    fs.writeFileSync(cliEnvFile, 'CLI_VAR=from_cli');
+    fs.writeFileSync(tempEnvFile, 'CONFIG_VAR=from_config');
+
+    fs.writeFileSync(
+      tempConfigFile,
+      `
+commandLineOptions:
+  envPath: ${tempEnvFile}
+
+prompts:
+  - "Test prompt"
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      input: "test"
+`,
+    );
+
+    const cmdObj = {
+      config: [tempConfigFile],
+      envPath: cliEnvFile,
+    };
+
+    try {
+      await doEval(cmdObj, {}, undefined, {});
+    } catch {
+      // Expected to fail due to mocked dependencies
+    }
+
+    // Should only call setupEnv once with CLI envPath (config envPath ignored)
+    expect(mockSetupEnv).toHaveBeenCalledTimes(1);
+    expect(mockSetupEnv).toHaveBeenCalledWith(cliEnvFile);
+  });
+
+  it('should handle missing commandLineOptions section gracefully', async () => {
+    fs.writeFileSync(
+      tempConfigFile,
+      `
+prompts:
+  - "Test prompt"
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      input: "test"
+`,
+    );
+
+    const cmdObj = { config: [tempConfigFile] };
+
+    try {
+      await doEval(cmdObj, {}, undefined, {});
+    } catch {
+      // Expected to fail due to mocked dependencies
+    }
+
+    // Should only call setupEnv once with undefined (no config envPath)
+    expect(mockSetupEnv).toHaveBeenCalledTimes(1);
+    expect(mockSetupEnv).toHaveBeenCalledWith(undefined);
+  });
+
+  it('should handle multiple config files and use first envPath found', async () => {
+    const config1File = path.join(tempDir, 'config1.yaml');
+    const config2File = path.join(tempDir, 'config2.yaml');
+    const envFile2 = path.join(tempDir, '.env2');
+
+    fs.writeFileSync(
+      config1File,
+      `
+prompts:
+  - "From config 1"
+`,
+    );
+
+    fs.writeFileSync(
+      config2File,
+      `
+commandLineOptions:
+  envPath: ${envFile2}
+
+prompts:
+  - "From config 2"
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      input: "test"
+`,
+    );
+
+    const cmdObj = { config: [config1File, config2File] };
+
+    try {
+      await doEval(cmdObj, {}, undefined, {});
+    } catch {
+      // Expected to fail due to mocked dependencies
+    }
+
+    // Should call setupEnv twice: CLI + first envPath found (from config2)
+    expect(mockSetupEnv).toHaveBeenCalledTimes(2);
+    expect(mockSetupEnv).toHaveBeenNthCalledWith(1, undefined);
+    expect(mockSetupEnv).toHaveBeenNthCalledWith(2, envFile2);
+  });
+
+  it('should resolve relative envPath against the config file directory', async () => {
+    const subDir = path.join(tempDir, 'sub');
+    fs.mkdirSync(subDir);
+    const relEnv = '.env.relative';
+    const relEnvAbs = path.join(subDir, relEnv);
+    fs.writeFileSync(relEnvAbs, 'REL=ok');
+
+    const subConfig = path.join(subDir, 'promptfooconfig.yaml');
+    fs.writeFileSync(
+      subConfig,
+      `
+commandLineOptions:
+  envPath: ${relEnv}
+
+prompts:
+  - "From sub config"
+providers:
+  - echo
+tests:
+  - vars: { input: "t" }
+`,
+    );
+
+    try {
+      await doEval({ config: [subConfig] }, {}, undefined, {});
+    } catch {}
+
+    expect(mockSetupEnv).toHaveBeenCalledTimes(2);
+    expect(mockSetupEnv).toHaveBeenNthCalledWith(1, undefined);
+    // Expect absolute resolved path
+    expect(path.isAbsolute((mockSetupEnv as any).mock.calls[1][0])).toBe(true);
+    expect((mockSetupEnv as any).mock.calls[1][0]).toBe(relEnvAbs);
+  });
+
+  describe('multi-file envPath support', () => {
+    it('should load multiple env files from config array', async () => {
+      const envFile1 = path.join(tempDir, '.env');
+      const envFile2 = path.join(tempDir, '.env.local');
+
+      fs.writeFileSync(envFile1, 'BASE_VAR=base');
+      fs.writeFileSync(envFile2, 'LOCAL_VAR=local');
+
+      fs.writeFileSync(
+        tempConfigFile,
+        dedentYaml`
+          commandLineOptions:
+            envPath:
+              - ${envFile1}
+              - ${envFile2}
+
+          prompts:
+            - "Test prompt"
+
+          providers:
+            - echo
+
+          tests:
+            - vars:
+                input: "test"
+        `,
+      );
+
+      const cmdObj = { config: [tempConfigFile] };
+
+      try {
+        await doEval(cmdObj, {}, undefined, {});
+      } catch {
+        // Expected to fail due to mocked dependencies
+      }
+
+      // Should call setupEnv twice: CLI (undefined), then config with array
+      expect(mockSetupEnv).toHaveBeenCalledTimes(2);
+      expect(mockSetupEnv).toHaveBeenNthCalledWith(1, undefined);
+      expect(mockSetupEnv).toHaveBeenNthCalledWith(2, [envFile1, envFile2]);
+    });
+
+    it('should resolve relative paths in envPath array against config directory', async () => {
+      const subDir = path.join(tempDir, 'nested');
+      fs.mkdirSync(subDir);
+
+      const envFile1 = path.join(subDir, '.env');
+      const envFile2 = path.join(subDir, '.env.local');
+
+      fs.writeFileSync(envFile1, 'VAR1=val1');
+      fs.writeFileSync(envFile2, 'VAR2=val2');
+
+      const subConfig = path.join(subDir, 'promptfooconfig.yaml');
+      fs.writeFileSync(
+        subConfig,
+        `
+commandLineOptions:
+  envPath:
+    - .env
+    - .env.local
+
+prompts:
+  - "Test"
+providers:
+  - echo
+tests:
+  - vars: { input: "t" }
+`,
+      );
+
+      try {
+        await doEval({ config: [subConfig] }, {}, undefined, {});
+      } catch {}
+
+      expect(mockSetupEnv).toHaveBeenCalledTimes(2);
+      expect(mockSetupEnv).toHaveBeenNthCalledWith(1, undefined);
+      // Should receive resolved absolute paths
+      const envPathArg = (mockSetupEnv as any).mock.calls[1][0];
+      expect(Array.isArray(envPathArg)).toBe(true);
+      expect(envPathArg).toEqual([envFile1, envFile2]);
+    });
+
+    it('should pass CLI envPath array when provided', async () => {
+      const envFile1 = path.join(tempDir, '.env.cli1');
+      const envFile2 = path.join(tempDir, '.env.cli2');
+
+      fs.writeFileSync(envFile1, 'CLI_VAR1=cli1');
+      fs.writeFileSync(envFile2, 'CLI_VAR2=cli2');
+
+      fs.writeFileSync(
+        tempConfigFile,
+        `
+prompts:
+  - "Test prompt"
+providers:
+  - echo
+tests:
+  - vars:
+      input: "test"
+`,
+      );
+
+      const cmdObj = {
+        config: [tempConfigFile],
+        envPath: [envFile1, envFile2],
+      };
+
+      try {
+        await doEval(cmdObj, {}, undefined, {});
+      } catch {}
+
+      // CLI envPath should be called once (no config envPath)
+      expect(mockSetupEnv).toHaveBeenCalledTimes(1);
+      expect(mockSetupEnv).toHaveBeenCalledWith([envFile1, envFile2]);
+    });
+
+    it('should load config envPath when CLI envPath defaults to an empty array', async () => {
+      const envFile1 = path.join(tempDir, '.env.empty-cli1');
+      const envFile2 = path.join(tempDir, '.env.empty-cli2');
+
+      fs.writeFileSync(envFile1, 'CONFIG_VAR1=config1');
+      fs.writeFileSync(envFile2, 'CONFIG_VAR2=config2');
+
+      fs.writeFileSync(
+        tempConfigFile,
+        dedentYaml`
+          commandLineOptions:
+            envPath:
+              - ${envFile1}
+              - ${envFile2}
+
+          prompts:
+            - "Test prompt"
+
+          providers:
+            - echo
+
+          tests:
+            - vars:
+                input: "test"
+        `,
+      );
+
+      try {
+        await doEval({ config: [tempConfigFile], envPath: [] }, {}, undefined, {});
+      } catch {}
+
+      expect(mockSetupEnv).toHaveBeenCalledTimes(2);
+      expect(mockSetupEnv).toHaveBeenNthCalledWith(1, []);
+      expect(mockSetupEnv).toHaveBeenNthCalledWith(2, [envFile1, envFile2]);
+    });
+  });
+});

test/integration/function-provider-grading.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Integration test to verify function providers work end-to-end in grading scenarios
+ * This tests the exact issue from #3784: function providers in defaultTest.options.provider
+ */
+import { describe, expect, it, vi } from 'vitest';
+import { getGradingProvider } from '../../src/matchers/providers';
+import { createMockProvider } from '../factories/provider';
+
+import type { ApiProvider, ProviderType } from '../../src/types/providers';
+
+describe('Function Provider Integration - Issue #3784', () => {
+  it('should work with getGradingProvider when passed as ApiProvider object', async () => {
+    // Create a mock function provider (simulating what resolveProvider returns)
+    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
+      return { output: `Graded: ${prompt}` };
+    });
+    mockFunctionProvider.label = 'test-grader';
+
+    // This is what resolveProvider returns for function providers.
+    // Use a plain object here because the test asserts callApi identity.
+    const resolvedProvider: ApiProvider = {
+      id: () => mockFunctionProvider.label,
+      callApi: mockFunctionProvider,
+    };
+
+    // Now pass it through getGradingProvider (this is what matcher provider helpers do)
+    const gradingProvider = await getGradingProvider(
+      'text' as ProviderType,
+      resolvedProvider,
+      null,
+    );
+
+    expect(gradingProvider).toBeDefined();
+    expect(gradingProvider).toBe(resolvedProvider); // Should return same object
+    expect(typeof gradingProvider!.id).toBe('function');
+    expect(gradingProvider!.id()).toBe('test-grader');
+    expect(gradingProvider!.callApi).toBe(mockFunctionProvider);
+  });
+
+  it('should actually call the function provider', async () => {
+    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
+      return { output: `Response for: ${prompt}` };
+    });
+
+    const resolvedProvider = createMockProvider({
+      id: 'custom-grader',
+      callApi: mockFunctionProvider,
+    });
+
+    const gradingProvider = await getGradingProvider(
+      'text' as ProviderType,
+      resolvedProvider,
+      null,
+    );
+
+    // Actually call the provider
+    const result = await gradingProvider!.callApi('test prompt');
+
+    expect(mockFunctionProvider).toHaveBeenCalledWith('test prompt');
+    expect(result.output).toBe('Response for: test prompt');
+  });
+
+  it('should handle function provider without label', async () => {
+    const mockFunctionProvider: any = vi.fn(async (prompt: string) => {
+      return { output: `Graded: ${prompt}` };
+    });
+
+    // No label, so resolveProvider uses 'custom-function'
+    const resolvedProvider = createMockProvider({
+      id: 'custom-function',
+      callApi: mockFunctionProvider,
+    });
+
+    const gradingProvider = await getGradingProvider(
+      'text' as ProviderType,
+      resolvedProvider,
+      null,
+    );
+
+    expect(gradingProvider).toBeDefined();
+    expect(gradingProvider!.id()).toBe('custom-function');
+  });
+
+  it('should correctly identify as ApiProvider based on type check', async () => {
+    const mockFunctionProvider: any = vi.fn(async () => ({ output: 'test' }));
+
+    const resolvedProvider = createMockProvider({
+      id: 'test',
+      callApi: mockFunctionProvider,
+    });
+
+    // This is the exact check from getGradingProvider line 120
+    const isApiProviderCheck =
+      typeof resolvedProvider === 'object' &&
+      typeof (resolvedProvider as ApiProvider).id === 'function';
+
+    expect(isApiProviderCheck).toBe(true);
+  });
+});

test/integration/library-exports.integration.test.ts

@@ -0,0 +1,201 @@
+/**
+ * Integration tests for library exports in both ESM and CJS formats.
+ *
+ * These tests verify that the built library can be imported correctly
+ * in both ESM and CJS environments after the build process.
+ *
+ * Prerequisites: Run `npm run build` before running these tests.
+ *
+ * NOTE: These tests are gated on a built `dist/` and become `describe.skip` when it is absent.
+ * The `integration-tests` CI job does not build, so they are skipped there and run only locally
+ * after `npm run build`. The published artifacts themselves are independently verified in CI by
+ * the build-and-pack `test:package-artifact` job (see scripts/testPackageArtifact.ts).
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import { beforeAll, describe, expect, it } from 'vitest';
+
+const distDir = path.resolve(__dirname, '../../dist/src');
+const buildExists = fs.existsSync(distDir);
+
+// Skip all tests if build doesn't exist (e.g., in CI Jest run before build)
+const describeIfBuildExists = buildExists ? describe : describe.skip;
+
+/**
+ * Follows the relative chunk imports out of a built entry file and returns the byte size of the
+ * full reachable local graph plus the set of bare (external) specifiers it pulls. Used to measure
+ * the contracts subpath's real footprint rather than just its thin re-export shim.
+ */
+function readModuleClosure(entryPath: string): { totalBytes: number; bareSpecifiers: Set<string> } {
+  const visited = new Set<string>();
+  const bareSpecifiers = new Set<string>();
+  let totalBytes = 0;
+  const stack = [entryPath];
+
+  while (stack.length > 0) {
+    const filePath = stack.pop() as string;
+    if (visited.has(filePath)) {
+      continue;
+    }
+    visited.add(filePath);
+
+    const source = fs.readFileSync(filePath, 'utf8');
+    totalBytes += Buffer.byteLength(source);
+
+    for (const match of source.matchAll(/(?:from|require|import)\s*\(?\s*['"]([^'"]+)['"]/g)) {
+      const specifier = match[1];
+      if (specifier.startsWith('.')) {
+        stack.push(path.resolve(path.dirname(filePath), specifier));
+      } else {
+        bareSpecifiers.add(specifier.replace(/^node:/, '').split('/')[0]);
+      }
+    }
+  }
+
+  return { totalBytes, bareSpecifiers };
+}
+
+describeIfBuildExists('Library Exports', () => {
+  describe('Build artifacts', () => {
+    it('should have ESM library build (index.js)', () => {
+      const esmPath = path.join(distDir, 'index.js');
+      expect(fs.existsSync(esmPath)).toBe(true);
+
+      const stats = fs.statSync(esmPath);
+      expect(stats.size).toBeGreaterThan(100000); // Should be substantial
+    });
+
+    it('should have CJS library build (index.cjs)', () => {
+      const cjsPath = path.join(distDir, 'index.cjs');
+      expect(fs.existsSync(cjsPath)).toBe(true);
+
+      const stats = fs.statSync(cjsPath);
+      expect(stats.size).toBeGreaterThan(100000); // Should be substantial
+    });
+
+    it('should have lightweight, zod-only contracts builds', () => {
+      // The contracts entry files are thin re-export shims; the real footprint lives in the shared
+      // chunks they import. Measure the whole transitive closure (entry + every reachable local
+      // chunk), not just the shim, so a regression that inlines a heavy dep into a chunk is caught.
+      for (const entry of ['contracts.js', 'contracts.cjs']) {
+        const { totalBytes, bareSpecifiers } = readModuleClosure(path.join(distDir, entry));
+        // ~14KB (ESM) / ~19KB (CJS) today; a heavy dep or inlined zod would blow well past this.
+        expect(totalBytes).toBeLessThan(50000);
+        // Leaf-safe contract: zod is the ONLY external the subpath may pull. This catches both a
+        // newly-leaked dependency (extra entry) AND zod accidentally being inlined (zod disappears).
+        expect([...bareSpecifiers].sort()).toEqual(['zod']);
+      }
+
+      for (const declaration of ['contracts.d.ts', 'contracts.d.cts']) {
+        const declarationPath = path.join(distDir, declaration);
+        expect(fs.existsSync(declarationPath)).toBe(true);
+        expect(fs.statSync(declarationPath).size).toBeLessThan(50000);
+      }
+    });
+
+    it('should have ESM CLI build (main.js)', () => {
+      const cliPath = path.join(distDir, 'main.js');
+      expect(fs.existsSync(cliPath)).toBe(true);
+
+      // CLI should have shebang
+      const content = fs.readFileSync(cliPath, 'utf8');
+      expect(content.startsWith('#!/usr/bin/env node')).toBe(true);
+    });
+
+    it('should have package.json with type: module in dist', () => {
+      const pkgPath = path.join(distDir, 'package.json');
+      expect(fs.existsSync(pkgPath)).toBe(true);
+
+      const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+      expect(pkg.type).toBe('module');
+    });
+  });
+
+  describe('CJS library import', () => {
+    let cjsModule: any;
+
+    beforeAll(() => {
+      // Use require to test CJS import
+      const cjsPath = path.join(distDir, 'index.cjs');
+      cjsModule = require(cjsPath);
+    });
+
+    it('should export key types and schemas', () => {
+      // Check for Zod schemas (common exports)
+      expect(cjsModule.AssertionSchema).toBeDefined();
+      expect(cjsModule.AtomicTestCaseSchema).toBeDefined();
+    });
+
+    it('should export utility functions', () => {
+      // These are commonly used exports
+      expect(typeof cjsModule.evaluate === 'function' || cjsModule.evaluate === undefined).toBe(
+        true,
+      );
+    });
+
+    it('should not throw when importing', () => {
+      expect(() => {
+        const cjsPath = path.join(distDir, 'index.cjs');
+        require(cjsPath);
+      }).not.toThrow();
+    });
+  });
+
+  describe('CJS contracts import', () => {
+    it('should export portable contract schemas', () => {
+      const contractsModule = require(path.join(distDir, 'contracts.cjs'));
+      expect(contractsModule.EmailSchema).toBeDefined();
+      expect(contractsModule.GetUserResponseSchema).toBeDefined();
+      expect(contractsModule.InputsSchema).toBeDefined();
+      expect(contractsModule.PromptSchema).toBeDefined();
+    });
+  });
+
+  describe('ESM library import', () => {
+    it('should be importable via dynamic import', async () => {
+      const esmPath = `file://${path.join(distDir, 'index.js')}`;
+      const esmModule = await import(esmPath);
+
+      // Check for Zod schemas (common exports)
+      expect(esmModule.AssertionSchema).toBeDefined();
+      expect(esmModule.AtomicTestCaseSchema).toBeDefined();
+    });
+
+    it('should export the same keys in ESM and CJS', async () => {
+      const esmPath = `file://${path.join(distDir, 'index.js')}`;
+      const cjsPath = path.join(distDir, 'index.cjs');
+
+      const esmModule = await import(esmPath);
+      const cjsModule = require(cjsPath);
+
+      const esmKeys = Object.keys(esmModule).sort();
+      const cjsKeys = Object.keys(cjsModule).sort();
+
+      // ESM may have additional 'default' export
+      const filteredEsmKeys = esmKeys.filter((k) => k !== 'default');
+
+      // The exports should be equivalent (allowing for minor differences)
+      expect(filteredEsmKeys.length).toBeGreaterThan(10); // Sanity check
+      expect(cjsKeys.length).toBeGreaterThan(10);
+
+      // Key exports should be present in both
+      const keyExports = ['AssertionSchema', 'AtomicTestCaseSchema', 'TestSuiteSchema'];
+      for (const key of keyExports) {
+        expect(filteredEsmKeys).toContain(key);
+        expect(cjsKeys).toContain(key);
+      }
+    });
+  });
+
+  describe('ESM contracts import', () => {
+    it('should export portable contract schemas', async () => {
+      const contractsModule = await import(`file://${path.join(distDir, 'contracts.js')}`);
+      expect(contractsModule.EmailSchema).toBeDefined();
+      expect(contractsModule.GetUserResponseSchema).toBeDefined();
+      expect(contractsModule.InputsSchema).toBeDefined();
+      expect(contractsModule.PromptSchema).toBeDefined();
+    });
+  });
+});

test/integration/tracing.test.ts

@@ -0,0 +1,293 @@
+/**
+ * Integration tests for OpenTelemetry tracing infrastructure.
+ *
+ * These tests verify that the OTEL SDK can be initialized and that
+ * provider calls correctly create spans with GenAI semantic conventions.
+ */
+
+import { SpanStatusCode } from '@opentelemetry/api';
+import { InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+  GenAIAttributes,
+  getGenAITracer,
+  PromptfooAttributes,
+  withGenAISpan,
+} from '../../src/tracing/genaiTracer';
+
+import type { GenAISpanContext, GenAISpanResult } from '../../src/tracing/genaiTracer';
+
+describe('OpenTelemetry Tracing Integration', () => {
+  let tracerProvider: NodeTracerProvider;
+  let memoryExporter: InMemorySpanExporter;
+
+  beforeAll(() => {
+    // Set up an in-memory exporter for testing
+    memoryExporter = new InMemorySpanExporter();
+    tracerProvider = new NodeTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(memoryExporter)],
+    });
+    tracerProvider.register();
+  });
+
+  afterAll(async () => {
+    await tracerProvider.shutdown();
+  });
+
+  beforeEach(() => {
+    memoryExporter.reset();
+    vi.resetAllMocks();
+  });
+
+  describe('withGenAISpan', () => {
+    it('should create a span with correct GenAI attributes', async () => {
+      const spanContext: GenAISpanContext = {
+        system: 'openai',
+        operationName: 'chat',
+        model: 'gpt-4',
+        providerId: 'openai:gpt-4',
+        maxTokens: 1000,
+        temperature: 0.7,
+        testIndex: 5,
+        promptLabel: 'test-prompt',
+      };
+
+      const mockResult = {
+        output: 'Hello, world!',
+        tokenUsage: { prompt: 10, completion: 5, total: 15 },
+      };
+
+      const resultExtractor = (): GenAISpanResult => ({
+        tokenUsage: { prompt: 10, completion: 5, total: 15 },
+        finishReasons: ['stop'],
+      });
+
+      const result = await withGenAISpan(spanContext, async () => mockResult, resultExtractor);
+
+      expect(result).toEqual(mockResult);
+
+      // Get the exported spans
+      const spans = memoryExporter.getFinishedSpans();
+      expect(spans.length).toBe(1);
+
+      const span = spans[0];
+
+      // Verify span name follows GenAI convention
+      expect(span.name).toBe('chat gpt-4');
+
+      // Verify GenAI attributes
+      expect(span.attributes[GenAIAttributes.SYSTEM]).toBe('openai');
+      expect(span.attributes[GenAIAttributes.OPERATION_NAME]).toBe('chat');
+      expect(span.attributes[GenAIAttributes.REQUEST_MODEL]).toBe('gpt-4');
+      expect(span.attributes[GenAIAttributes.REQUEST_MAX_TOKENS]).toBe(1000);
+      expect(span.attributes[GenAIAttributes.REQUEST_TEMPERATURE]).toBe(0.7);
+
+      // Verify Promptfoo attributes
+      expect(span.attributes[PromptfooAttributes.PROVIDER_ID]).toBe('openai:gpt-4');
+      expect(span.attributes[PromptfooAttributes.TEST_INDEX]).toBe(5);
+      expect(span.attributes[PromptfooAttributes.PROMPT_LABEL]).toBe('test-prompt');
+
+      // Verify response attributes
+      expect(span.attributes[GenAIAttributes.USAGE_INPUT_TOKENS]).toBe(10);
+      expect(span.attributes[GenAIAttributes.USAGE_OUTPUT_TOKENS]).toBe(5);
+      expect(span.attributes[GenAIAttributes.USAGE_TOTAL_TOKENS]).toBe(15);
+      expect(span.attributes[GenAIAttributes.RESPONSE_FINISH_REASONS]).toEqual(['stop']);
+
+      // Verify span status
+      expect(span.status.code).toBe(SpanStatusCode.OK);
+    });
+
+    it('should handle errors and set span status to ERROR', async () => {
+      const spanContext: GenAISpanContext = {
+        system: 'anthropic',
+        operationName: 'chat',
+        model: 'claude-3-opus',
+        providerId: 'anthropic:claude-3-opus',
+      };
+
+      const error = new Error('API rate limit exceeded');
+
+      await expect(
+        withGenAISpan(spanContext, async () => {
+          throw error;
+        }),
+      ).rejects.toThrow('API rate limit exceeded');
+
+      const spans = memoryExporter.getFinishedSpans();
+      expect(spans.length).toBe(1);
+
+      const span = spans[0];
+
+      // Verify span status is ERROR
+      expect(span.status.code).toBe(SpanStatusCode.ERROR);
+      expect(span.status.message).toBe('API rate limit exceeded');
+
+      // Verify exception was recorded
+      expect(span.events.length).toBeGreaterThan(0);
+      const exceptionEvent = span.events.find((e) => e.name === 'exception');
+      expect(exceptionEvent).toBeDefined();
+    });
+
+    it('should work without result extractor', async () => {
+      const spanContext: GenAISpanContext = {
+        system: 'bedrock',
+        operationName: 'chat',
+        model: 'anthropic.claude-3-sonnet',
+        providerId: 'bedrock:claude-3-sonnet',
+      };
+
+      const result = await withGenAISpan(spanContext, async () => ({
+        output: 'response',
+      }));
+
+      expect(result).toEqual({ output: 'response' });
+
+      const spans = memoryExporter.getFinishedSpans();
+      expect(spans.length).toBe(1);
+
+      // Should still have basic attributes
+      expect(spans[0].attributes[GenAIAttributes.SYSTEM]).toBe('bedrock');
+      expect(spans[0].status.code).toBe(SpanStatusCode.OK);
+    });
+
+    it('should capture multiple nested spans correctly', async () => {
+      const outerContext: GenAISpanContext = {
+        system: 'azure',
+        operationName: 'chat',
+        model: 'gpt-4-deployment',
+        providerId: 'azure:gpt-4',
+      };
+
+      const innerContext: GenAISpanContext = {
+        system: 'openai',
+        operationName: 'embedding',
+        model: 'text-embedding-ada-002',
+        providerId: 'openai:embedding',
+      };
+
+      await withGenAISpan(outerContext, async () => {
+        // Nested span for embedding
+        await withGenAISpan(innerContext, async () => {
+          return { embedding: [0.1, 0.2, 0.3] };
+        });
+        return { output: 'response' };
+      });
+
+      const spans = memoryExporter.getFinishedSpans();
+      expect(spans.length).toBe(2);
+
+      // Inner span should finish first
+      const embeddingSpan = spans.find((s) => s.name.includes('embedding'));
+      const chatSpan = spans.find((s) => s.name.includes('chat'));
+
+      expect(embeddingSpan).toBeDefined();
+      expect(chatSpan).toBeDefined();
+      expect(embeddingSpan!.attributes[GenAIAttributes.SYSTEM]).toBe('openai');
+      expect(chatSpan!.attributes[GenAIAttributes.SYSTEM]).toBe('azure');
+    });
+  });
+
+  describe('getGenAITracer', () => {
+    it('should return a tracer with correct name', () => {
+      const tracer = getGenAITracer();
+      expect(tracer).toBeDefined();
+      // Tracer should be usable
+      const span = tracer.startSpan('test-span');
+      span.end();
+
+      const spans = memoryExporter.getFinishedSpans();
+      expect(spans.some((s) => s.name === 'test-span')).toBe(true);
+    });
+  });
+
+  describe('Token usage with completion details', () => {
+    it('should capture reasoning tokens in completion details', async () => {
+      const spanContext: GenAISpanContext = {
+        system: 'openai',
+        operationName: 'chat',
+        model: 'o1-preview',
+        providerId: 'openai:o1-preview',
+      };
+
+      const resultExtractor = (): GenAISpanResult => ({
+        tokenUsage: {
+          prompt: 100,
+          completion: 500,
+          total: 600,
+          completionDetails: {
+            reasoning: 450,
+          },
+        },
+      });
+
+      await withGenAISpan(spanContext, async () => ({ output: 'response' }), resultExtractor);
+
+      const spans = memoryExporter.getFinishedSpans();
+      expect(spans.length).toBe(1);
+
+      const span = spans[0];
+      expect(span.attributes[GenAIAttributes.USAGE_INPUT_TOKENS]).toBe(100);
+      expect(span.attributes[GenAIAttributes.USAGE_OUTPUT_TOKENS]).toBe(500);
+      expect(span.attributes[GenAIAttributes.USAGE_REASONING_TOKENS]).toBe(450);
+    });
+
+    it('should capture predicted token details', async () => {
+      const spanContext: GenAISpanContext = {
+        system: 'openai',
+        operationName: 'chat',
+        model: 'gpt-4-turbo',
+        providerId: 'openai:gpt-4-turbo',
+      };
+
+      const resultExtractor = (): GenAISpanResult => ({
+        tokenUsage: {
+          prompt: 50,
+          completion: 30,
+          total: 80,
+          completionDetails: {
+            acceptedPrediction: 25,
+            rejectedPrediction: 5,
+          },
+        },
+      });
+
+      await withGenAISpan(spanContext, async () => ({ output: 'response' }), resultExtractor);
+
+      const spans = memoryExporter.getFinishedSpans();
+      const span = spans[0];
+
+      expect(span.attributes[GenAIAttributes.USAGE_ACCEPTED_PREDICTION_TOKENS]).toBe(25);
+      expect(span.attributes[GenAIAttributes.USAGE_REJECTED_PREDICTION_TOKENS]).toBe(5);
+    });
+
+    it('should capture cached tokens', async () => {
+      const spanContext: GenAISpanContext = {
+        system: 'anthropic',
+        operationName: 'chat',
+        model: 'claude-3-sonnet',
+        providerId: 'anthropic:claude-3-sonnet',
+      };
+
+      const resultExtractor = (): GenAISpanResult => ({
+        tokenUsage: {
+          prompt: 200,
+          completion: 100,
+          total: 300,
+          cached: 150,
+        },
+      });
+
+      await withGenAISpan(
+        spanContext,
+        async () => ({ output: 'cached response' }),
+        resultExtractor,
+      );
+
+      const spans = memoryExporter.getFinishedSpans();
+      const span = spans[0];
+
+      expect(span.attributes[GenAIAttributes.USAGE_CACHED_TOKENS]).toBe(150);
+    });
+  });
+});