chore: import upstream snapshot with attribution

test/smoke/advanced-features.test.ts

@@ -0,0 +1,379 @@
+/**
+ * Smoke tests for advanced CLI features.
+ *
+ * These tests verify advanced evaluation features including:
+ * - Environment file loading (--env-file)
+ * - Delay between tests (--delay)
+ * - HTML output format
+ * - Config loading from separate files
+ * - Multiple prompts (prompt comparison/A-B testing)
+ * - icontains assertion (case-insensitive)
+ * - regex assertion with end-of-string pattern
+ * - Multiple file prompts (file:// references)
+ * - Assertion weights
+ * - Test threshold option
+ *
+ * @see docs/plans/smoke-tests.md for the full checklist
+ */
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+// Path to the built CLI binary
+const CLI_PATH = path.resolve(__dirname, '../../dist/src/main.js');
+const ROOT_DIR = path.resolve(__dirname, '../..');
+const FIXTURES_DIR = path.resolve(__dirname, 'fixtures');
+const OUTPUT_DIR = path.resolve(__dirname, '.temp-output-advanced');
+
+/**
+ * Helper to run the CLI and capture output
+ */
+function runCli(
+  args: string[],
+  options: { cwd?: string; expectError?: boolean; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [CLI_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 60000,
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+describe('Advanced Features Smoke Tests', () => {
+  beforeAll(() => {
+    // Verify the built binary exists
+    if (!fs.existsSync(CLI_PATH)) {
+      throw new Error(`Built CLI not found at ${CLI_PATH}. Run 'npm run build' first.`);
+    }
+
+    // Create output directory for test artifacts
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    // Clean up output directory
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('1.4.8 Environment File Loading', () => {
+    it('1.4.8 - --env-file loads environment variables', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/env-var-test.yaml');
+      const envFilePath = path.join(FIXTURES_DIR, 'data/test.env');
+      const outputPath = path.join(OUTPUT_DIR, 'env-test-output.json');
+
+      const { exitCode } = runCli([
+        'eval',
+        '-c',
+        configPath,
+        '--env-file',
+        envFilePath,
+        '-o',
+        outputPath,
+        '--no-cache',
+      ]);
+
+      expect(exitCode).toBe(0);
+
+      // Verify the env vars were substituted in the output
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const output = parsed.results.results[0].response.output;
+
+      // The env vars should be present in the echoed output
+      expect(output).toContain('sk-test-12345');
+      expect(output).toContain('super-secret-value');
+    });
+
+    it('1.4.8b - --env-file resolves env placeholders in file:// provider references', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/file-provider-env-7079.yaml');
+      const envFilePath = path.join(FIXTURES_DIR, 'data/file-provider-env-7079.env');
+      const outputPath = path.join(OUTPUT_DIR, 'file-provider-env-7079-output.json');
+
+      const { exitCode } = runCli([
+        'eval',
+        '-c',
+        configPath,
+        '--env-file',
+        envFilePath,
+        '-o',
+        outputPath,
+        '--no-cache',
+      ]);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const result = parsed.results.results[0];
+
+      expect(result.provider.id).toBe('echo');
+      expect(result.provider.label).toBe('Smoke Provider Label');
+      expect(result.response.output).toContain('Hello World');
+    });
+  });
+
+  describe('2.5.2 Conversation Relevance Config Loading', () => {
+    it('2.5.2 - preserves template syntax in static _conversation vars', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/conversation-relevance-ssti.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'conversation-relevance-ssti-output.json');
+      const capturePath = path.join(OUTPUT_DIR, 'conversation-relevance-ssti-prompt.txt');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        env: {
+          OPENAI_API_KEY: 'SHOULD_NOT_RENDER',
+          PROMPTFOO_CAPTURE_PATH: capturePath,
+        },
+      });
+
+      expect(exitCode).toBe(0);
+
+      const prompt = fs.readFileSync(capturePath, 'utf-8');
+      expect(prompt).toContain('What is the capital of {{country}}?');
+      expect(prompt).toContain('{{capital}}');
+      expect(prompt).toContain('{{ env.OPENAI_API_KEY }}');
+      expect(prompt).toContain('{% for x in range(3) %}{{x}}{% endfor %}');
+      expect(prompt).not.toContain('SHOULD_NOT_RENDER');
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+  });
+
+  describe('1.10.1 Delay Between Tests', () => {
+    it('1.10.1 - --delay adds delay between tests', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/delay-test.yaml');
+
+      // Run with 100ms delay between 3 tests
+      const startTime = Date.now();
+      const { exitCode } = runCli(['eval', '-c', configPath, '--delay', '100', '--no-cache']);
+      const endTime = Date.now();
+
+      expect(exitCode).toBe(0);
+
+      // With 3 tests and 100ms delay, total time should be at least 200ms (2 delays)
+      // Being generous with timing to avoid flaky tests
+      const elapsed = endTime - startTime;
+      expect(elapsed).toBeGreaterThanOrEqual(150); // At least some delay occurred
+    });
+  });
+
+  describe('1.4.4b HTML Output Format', () => {
+    it('1.4.4b - outputs HTML format', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'output.html');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+      expect(fs.existsSync(outputPath)).toBe(true);
+
+      // Verify it's HTML content (lowercase doctype is valid HTML5)
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      expect(content).toContain('<!doctype html>');
+      expect(content).toContain('<html');
+    });
+  });
+
+  describe('2.5 Config Loading', () => {
+    it('2.5.1 - loads separate config file correctly', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/config-tests.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'config-load-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Verify the config loaded and ran correctly
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const output = parsed.results.results[0].response.output;
+
+      expect(output).toContain('ConfigMergeTest');
+    });
+  });
+
+  describe('4.3.1b Multiple Prompts', () => {
+    it('4.3.1b - evaluates multiple prompts against same tests', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/multi-prompt.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'multi-prompt-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Verify we have results for all 3 prompts
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have 3 prompts
+      expect(parsed.results.prompts.length).toBe(3);
+
+      // Should have 3 results (one test case x 3 prompts)
+      expect(parsed.results.results.length).toBe(3);
+
+      // Each prompt should produce different output
+      const outputs = parsed.results.results.map(
+        (r: { response: { output: string } }) => r.response.output,
+      );
+      expect(outputs.some((o: string) => o.includes('Hello'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Hi'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Hey'))).toBe(true);
+    });
+  });
+
+  describe('5.1.2b icontains Assertion', () => {
+    it('5.1.2b - icontains performs case-insensitive match', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/icontains-assertion.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'icontains-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // All case-insensitive assertions should pass
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.results.results[0].success).toBe(true);
+      // Should have 3 assertions all passing
+      expect(parsed.results.results[0].gradingResult.componentResults.length).toBe(3);
+      expect(
+        parsed.results.results[0].gradingResult.componentResults.every(
+          (r: { pass: boolean }) => r.pass,
+        ),
+      ).toBe(true);
+    });
+  });
+
+  describe('5.1.5b Regex End-of-String Pattern', () => {
+    it('5.1.5b - regex with $ anchor checks string suffix', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/ends-with-assertion.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'regex-ends-with-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Both regex end-of-string assertions should pass
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.results.results[0].success).toBe(true);
+      expect(parsed.results.results[0].gradingResult.componentResults.length).toBe(2);
+    });
+  });
+
+  describe('4.3.2b Multiple File Prompts', () => {
+    it('4.3.2b - file:// references load multiple prompt files', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/glob-prompts.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'file-prompts-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Should load both prompt files from file:// references
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have 2 prompts (greeting.txt and farewell.txt)
+      expect(parsed.results.prompts.length).toBe(2);
+
+      // Should have 2 results (1 test x 2 prompts)
+      expect(parsed.results.results.length).toBe(2);
+
+      // Verify both prompts were used
+      const outputs = parsed.results.results.map(
+        (r: { response: { output: string } }) => r.response.output,
+      );
+      expect(outputs.some((o: string) => o.includes('welcome'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('see you'))).toBe(true);
+    });
+  });
+
+  describe('5.3 Assertion Weights', () => {
+    it('5.3.1 - assertion weights affect scoring', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/weighted-assertions.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'weighted-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Both assertions pass, so test passes
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.results.results[0].success).toBe(true);
+      // The weighted assertions should both pass
+      expect(parsed.results.results[0].gradingResult.componentResults.length).toBe(2);
+    });
+
+    it('5.3.2 - weighted named metrics preserve prompt totals and denominators', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/weighted-named-metric.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'weighted-named-metric-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      expect(parsed.results.results[0].success).toBe(true);
+      expect(parsed.results.results[0].score).toBeCloseTo(0.75, 10);
+      expect(parsed.results.results[0].gradingResult.namedScores.weightedMetric).toBeCloseTo(
+        0.75,
+        10,
+      );
+      expect(parsed.results.results[0].gradingResult.namedScoreWeights.weightedMetric).toBe(4);
+      expect(parsed.results.prompts[0].metrics.namedScores.weightedMetric).toBeCloseTo(3, 10);
+      expect(parsed.results.prompts[0].metrics.namedScoresCount.weightedMetric).toBe(2);
+      expect(parsed.results.prompts[0].metrics.namedScoreWeights.weightedMetric).toBe(4);
+      expect(
+        parsed.results.prompts[0].metrics.namedScores.weightedMetric /
+          parsed.results.prompts[0].metrics.namedScoreWeights.weightedMetric,
+      ).toBeCloseTo(0.75, 10);
+    });
+  });
+
+  describe('7.4 Test Threshold', () => {
+    it('7.4.1 - test threshold allows partial assertion passes', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/test-threshold.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'threshold-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      // The test should pass because 2/3 assertions pass (66%) > threshold (50%)
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Test should pass due to threshold
+      expect(parsed.results.results[0].success).toBe(true);
+
+      // Verify assertion results
+      const componentResults = parsed.results.results[0].gradingResult.componentResults;
+      expect(componentResults.length).toBe(3);
+
+      // 2 should pass, 1 should fail
+      const passCount = componentResults.filter((r: { pass: boolean }) => r.pass).length;
+      const failCount = componentResults.filter((r: { pass: boolean }) => !r.pass).length;
+      expect(passCount).toBe(2);
+      expect(failCount).toBe(1);
+    });
+  });
+});

test/smoke/cli.test.ts

@@ -0,0 +1,438 @@
+/**
+ * Smoke tests for CLI binary operations.
+ *
+ * These tests verify the built CLI binary works correctly.
+ * They run against dist/src/main.js (the built package).
+ *
+ * @see docs/plans/smoke-tests.md for the full checklist
+ */
+import { spawn, spawnSync } from 'child_process';
+import * as fs from 'fs';
+import { createServer, type Server as HttpServer } from 'http';
+import * as path from 'path';
+
+import { Server as SocketIOServer } from 'socket.io';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+// Path to the built CLI binaries
+const CLI_PATH = path.resolve(__dirname, '../../dist/src/main.js');
+const ENTRYPOINT_PATH = path.resolve(__dirname, '../../dist/src/entrypoint.js');
+const ROOT_DIR = path.resolve(__dirname, '../..');
+
+/**
+ * Helper to run the CLI and capture output
+ */
+function runCli(
+  args: string[],
+  options: { cwd?: string; expectError?: boolean; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [CLI_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 30000,
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+/**
+ * Helper to run the CLI via entrypoint.js (the actual bin entry)
+ */
+function runEntrypoint(
+  args: string[],
+  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [ENTRYPOINT_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 30000,
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+function runEntrypointAsync(
+  args: string[],
+  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
+): Promise<{ stdout: string; stderr: string; exitCode: number }> {
+  return new Promise((resolve, reject) => {
+    const child = spawn('node', [ENTRYPOINT_PATH, ...args], {
+      cwd: options.cwd || ROOT_DIR,
+      env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    });
+
+    let stdout = '';
+    let stderr = '';
+    const timeoutId = setTimeout(() => {
+      child.kill('SIGTERM');
+      reject(new Error(`CLI entrypoint timed out for: ${args.join(' ')}`));
+    }, 30000);
+
+    child.stdout.setEncoding('utf8');
+    child.stdout.on('data', (chunk: string) => {
+      stdout += chunk;
+    });
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', (chunk: string) => {
+      stderr += chunk;
+    });
+    child.once('error', (error) => {
+      clearTimeout(timeoutId);
+      reject(error);
+    });
+    child.once('close', (code) => {
+      clearTimeout(timeoutId);
+      resolve({
+        stdout,
+        stderr,
+        exitCode: code ?? 1,
+      });
+    });
+  });
+}
+
+function runGit(args: string[], cwd: string): void {
+  const result = spawnSync('git', args, {
+    cwd,
+    encoding: 'utf-8',
+  });
+
+  if (result.status !== 0) {
+    throw new Error(
+      `git ${args.join(' ')} failed: ${(result.stderr || result.stdout || '').trim()}`,
+    );
+  }
+}
+
+describe('CLI Smoke Tests', () => {
+  beforeAll(() => {
+    // Verify the built binaries exist
+    if (!fs.existsSync(CLI_PATH)) {
+      throw new Error(`Built CLI not found at ${CLI_PATH}. Run 'npm run build' first.`);
+    }
+    if (!fs.existsSync(ENTRYPOINT_PATH)) {
+      throw new Error(
+        `Built entrypoint not found at ${ENTRYPOINT_PATH}. Run 'npm run build' first.`,
+      );
+    }
+  });
+
+  describe('1.1 Basic CLI Operations', () => {
+    it('1.1.1 - outputs version with --version', () => {
+      const { stdout, exitCode } = runCli(['--version']);
+
+      expect(exitCode).toBe(0);
+      // Version should contain semver pattern (e.g., 1.2.3 or 1.2.3-beta.1)
+      // Note: Output may include warnings if invalid config files exist in cwd
+      expect(stdout).toMatch(/\d+\.\d+\.\d+(-[\w.]+)?/);
+    });
+
+    it('1.1.2 - outputs help with --help', () => {
+      const { stdout, exitCode } = runCli(['--help']);
+
+      expect(exitCode).toBe(0);
+      expect(stdout).toContain('promptfoo');
+      expect(stdout).toContain('eval');
+      expect(stdout).toContain('Commands:');
+    });
+
+    it('1.1.3 - outputs subcommand help with eval --help', () => {
+      const { stdout, exitCode } = runCli(['eval', '--help']);
+
+      expect(exitCode).toBe(0);
+      expect(stdout).toContain('--config');
+      expect(stdout).toContain('--output');
+      expect(stdout).toContain('--no-cache');
+    });
+
+    it('1.1.4 - handles unknown command gracefully', () => {
+      const { stderr, exitCode } = runCli(['unknowncommand123']);
+
+      expect(exitCode).toBe(1);
+      // Commander outputs "error: unknown command" (lowercase)
+      expect(stderr.toLowerCase()).toContain('unknown command');
+    });
+
+    it('1.1.5 - handles missing config file gracefully', () => {
+      const { stdout, stderr, exitCode } = runCli(['eval', '-c', 'nonexistent-config-file.yaml']);
+
+      expect(exitCode).toBe(1);
+      // Error may appear in stdout (uncaught exception) or stderr
+      const output = stdout + stderr;
+      // Should indicate the file wasn't found
+      expect(output.toLowerCase()).toMatch(
+        /not found|no such file|does not exist|cannot find|no configuration file/i,
+      );
+    });
+  });
+
+  describe('1.2 Init Command', () => {
+    let tempDir: string;
+
+    beforeAll(() => {
+      // Create a temp directory for init tests
+      tempDir = path.join(ROOT_DIR, 'test/smoke/.temp-init-test');
+      fs.mkdirSync(tempDir, { recursive: true });
+    });
+
+    afterAll(() => {
+      // Clean up temp directory
+      if (fs.existsSync(tempDir)) {
+        fs.rmSync(tempDir, { recursive: true, force: true });
+      }
+    });
+
+    it('1.2.1 - init --no-interactive creates config file', () => {
+      const initDir = path.join(tempDir, 'init-test');
+      fs.mkdirSync(initDir, { recursive: true });
+
+      const { exitCode } = runCli(['init', '--no-interactive'], { cwd: initDir });
+
+      expect(exitCode).toBe(0);
+      // Should create a promptfooconfig file
+      const configExists =
+        fs.existsSync(path.join(initDir, 'promptfooconfig.yaml')) ||
+        fs.existsSync(path.join(initDir, 'promptfooconfig.yml'));
+      expect(configExists).toBe(true);
+    });
+  });
+
+  describe('1.3 Validate Command', () => {
+    const fixturesDir = path.join(__dirname, 'fixtures/configs');
+
+    it('1.3.1 - validates a correct config file', () => {
+      const { stdout, exitCode } = runCli(['validate', '-c', path.join(fixturesDir, 'basic.yaml')]);
+
+      expect(exitCode).toBe(0);
+      expect(stdout.toLowerCase()).toContain('valid');
+    });
+
+    it('1.3.2 - rejects an invalid config file', () => {
+      const { stdout, stderr, exitCode } = runCli([
+        'validate',
+        '-c',
+        path.join(fixturesDir, 'invalid.yaml'),
+      ]);
+
+      expect(exitCode).toBe(1);
+      // Error may appear in stdout or stderr
+      const output = stdout + stderr;
+      expect(output.length).toBeGreaterThan(0);
+      // Should indicate a validation error (missing providers)
+      expect(output.toLowerCase()).toMatch(/provider|invalid|error/i);
+    });
+  });
+
+  describe('1.6 Cache Commands', () => {
+    it('1.6.1 - cache clear executes without error', () => {
+      const { exitCode } = runCli(['cache', 'clear']);
+
+      // Should succeed (exit 0) even if cache is empty
+      expect(exitCode).toBe(0);
+    });
+  });
+
+  describe('1.5 List Commands', () => {
+    it('1.5.1 - list evals executes without error', () => {
+      const { exitCode } = runCli(['list', 'evals']);
+
+      // Should succeed even if no evals exist
+      expect(exitCode).toBe(0);
+    });
+
+    it('1.5.2 - list datasets executes without error', () => {
+      const { exitCode } = runCli(['list', 'datasets']);
+
+      // Should succeed even if no datasets exist
+      expect(exitCode).toBe(0);
+    });
+  });
+
+  describe('1.7 Entrypoint Wrapper', () => {
+    /**
+     * These tests verify that entrypoint.js (the actual bin entry) works correctly.
+     * The entrypoint provides a Node.js version check before loading dependencies.
+     */
+
+    it('1.7.1 - entrypoint outputs version with --version', () => {
+      const { stdout, exitCode } = runEntrypoint(['--version']);
+
+      expect(exitCode).toBe(0);
+      // Version should contain semver pattern
+      expect(stdout).toMatch(/\d+\.\d+\.\d+(-[\w.]+)?/);
+    });
+
+    it('1.7.2 - entrypoint outputs help with --help', () => {
+      const { stdout, exitCode } = runEntrypoint(['--help']);
+
+      expect(exitCode).toBe(0);
+      expect(stdout).toContain('promptfoo');
+      expect(stdout).toContain('eval');
+      expect(stdout).toContain('Commands:');
+    });
+
+    it('1.7.3 - entrypoint handles subcommands correctly', () => {
+      const { stdout, exitCode } = runEntrypoint(['eval', '--help']);
+
+      expect(exitCode).toBe(0);
+      expect(stdout).toContain('--config');
+      expect(stdout).toContain('--output');
+    });
+
+    it('1.7.4 - entrypoint produces identical output to main.js', () => {
+      // Verify entrypoint delegates to main.js correctly
+      const entrypointResult = runEntrypoint(['--version']);
+      const mainResult = runCli(['--version']);
+
+      expect(entrypointResult.exitCode).toBe(mainResult.exitCode);
+      expect(entrypointResult.stdout.trim()).toBe(mainResult.stdout.trim());
+    });
+  });
+
+  describe('1.8 Code Scan Structured Output', () => {
+    let repoDir: string;
+    let httpServer: HttpServer;
+    let socketServer: SocketIOServer;
+    let apiHost: string;
+
+    beforeAll(async () => {
+      repoDir = path.join(ROOT_DIR, 'test/smoke/.temp-code-scan-structured-output');
+      fs.rmSync(repoDir, { recursive: true, force: true });
+      fs.mkdirSync(repoDir, { recursive: true });
+
+      runGit(['init', '-b', 'main'], repoDir);
+      runGit(['config', 'user.email', 'smoke@example.com'], repoDir);
+      runGit(['config', 'user.name', 'Promptfoo Smoke Test'], repoDir);
+      runGit(['commit', '--allow-empty', '-m', 'baseline'], repoDir);
+
+      httpServer = createServer((_request, response) => {
+        response.writeHead(200, { 'content-type': 'text/plain' });
+        response.end('ok');
+      });
+      socketServer = new SocketIOServer(httpServer, {
+        cors: { origin: '*' },
+      });
+      socketServer.on('connection', (socket) => {
+        socket.on('agent:join', () => {});
+      });
+
+      await new Promise<void>((resolve, reject) => {
+        httpServer.once('error', reject);
+        httpServer.listen(0, '127.0.0.1', () => {
+          httpServer.off('error', reject);
+          const address = httpServer.address();
+          if (!address || typeof address === 'string') {
+            reject(new Error('Failed to resolve mock code-scan server port'));
+            return;
+          }
+          apiHost = `http://127.0.0.1:${address.port}`;
+          resolve();
+        });
+      });
+    });
+
+    afterAll(async () => {
+      await new Promise<void>((resolve) => {
+        socketServer.close(() => {
+          httpServer.close(() => resolve());
+        });
+      });
+      fs.rmSync(repoDir, { recursive: true, force: true });
+    });
+
+    it.each([
+      [
+        'JSON',
+        ['--json'],
+        (payload: Record<string, unknown>) => {
+          expect(payload).toMatchObject({
+            success: true,
+            comments: [],
+            review: 'No files to scan',
+          });
+        },
+      ],
+      [
+        'SARIF',
+        ['--format', 'sarif'],
+        (payload: Record<string, unknown>) => {
+          expect(payload).toMatchObject({
+            version: '2.1.0',
+            runs: [expect.any(Object)],
+          });
+        },
+      ],
+    ])('1.8.%# - keeps %s stdout machine-readable under --verbose and LOG_LEVEL=debug', async (_label, outputArgs, assertPayload) => {
+      const { stdout, stderr, exitCode } = await runEntrypointAsync(
+        [
+          'code-scans',
+          'run',
+          repoDir,
+          '--diffs-only',
+          '--api-host',
+          apiHost,
+          '--base',
+          'main',
+          '--compare',
+          'HEAD',
+          ...outputArgs,
+          '--verbose',
+        ],
+        {
+          env: {
+            LOG_LEVEL: 'debug',
+            NODE_NO_WARNINGS: '1',
+            PROMPTFOO_DISABLE_UPDATE: 'true',
+          },
+        },
+      );
+
+      expect(exitCode).toBe(0);
+      expect(stderr).toBe('');
+
+      const payload = JSON.parse(stdout) as Record<string, unknown>;
+      assertPayload(payload);
+    });
+
+    it('1.8.2 - keeps structured config failures off stdout', async () => {
+      const missingConfigPath = path.join(repoDir, 'definitely-missing-code-scan-config.yaml');
+      const { stdout, stderr, exitCode } = await runEntrypointAsync(
+        ['code-scans', 'run', repoDir, '--json', '--config', missingConfigPath],
+        {
+          env: {
+            NODE_NO_WARNINGS: '1',
+          },
+        },
+      );
+
+      expect(exitCode).toBe(1);
+      expect(stdout).toBe('');
+      expect(stderr).toContain(`Scan failed: Configuration file not found: ${missingConfigPath}`);
+    });
+
+    it('1.8.3 - keeps structured output flag conflicts off stdout', async () => {
+      const { stdout, stderr, exitCode } = await runEntrypointAsync(
+        ['code-scans', 'run', repoDir, '--json', '--format', 'sarif'],
+        {
+          env: {
+            NODE_NO_WARNINGS: '1',
+          },
+        },
+      );
+
+      expect(exitCode).toBe(1);
+      expect(stdout).toBe('');
+      expect(stderr).toContain('Scan failed: Cannot combine --json with --format sarif');
+    });
+  });
+});

test/smoke/configs-and-providers.test.ts

@@ -0,0 +1,398 @@
+/**
+ * Smoke tests for config formats, providers, and data loading.
+ *
+ * These tests verify various configuration formats, provider types,
+ * and data loading mechanisms work correctly.
+ *
+ * @see docs/plans/smoke-tests.md for the full checklist
+ */
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { getOpenAiMissingApiKeyMessage } from '../providers/openai/shared';
+
+// Path to the built CLI binary
+const CLI_PATH = path.resolve(__dirname, '../../dist/src/main.js');
+const ROOT_DIR = path.resolve(__dirname, '../..');
+const FIXTURES_DIR = path.resolve(__dirname, 'fixtures');
+const OUTPUT_DIR = path.resolve(__dirname, '.temp-output-configs');
+
+/**
+ * Helper to run the CLI and capture output
+ */
+function runCli(
+  args: string[],
+  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [CLI_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 60000,
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+describe('Config Format Smoke Tests', () => {
+  beforeAll(() => {
+    if (!fs.existsSync(CLI_PATH)) {
+      throw new Error(`Built CLI not found at ${CLI_PATH}. Run 'npm run build' first.`);
+    }
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('2.2 JSON Configs', () => {
+    it('2.2.1 - parses JSON config format', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.json');
+      const outputPath = path.join(OUTPUT_DIR, 'json-config-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Verify the eval ran correctly
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+  });
+});
+
+describe('Provider Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('3.1 Built-in Providers', () => {
+    it('3.1.2 - exec provider executes shell commands', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/exec-provider.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'exec-provider-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const result = parsed.results.results[0];
+
+      // Exec provider should have executed the echo command
+      expect(result.response.output).toContain('Echo from exec');
+    });
+
+    // Regression for #8686: exec provider must close the child's stdin so that
+    // scripts reading from stdin (e.g. opencode) do not hang forever.
+    it('3.1.3 - exec provider closes stdin on child process', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/exec-provider-stdin.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'exec-provider-stdin-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const result = parsed.results.results[0];
+
+      expect(result.success).toBe(true);
+      expect(result.response.output).toContain('stdin closed');
+    });
+  });
+
+  describe('3.3 OpenAI Providers', () => {
+    it('3.3.1 - passes tools and tool_choice to providers', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/normalized-tools.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'normalized-tools-output.json');
+
+      // Run from configs directory so relative script paths work
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: path.join(FIXTURES_DIR, 'configs'),
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have results for all 3 providers
+      expect(parsed.results.results.length).toBe(3);
+
+      // All results should be successful
+      parsed.results.results.forEach((result: { success: boolean }) => {
+        expect(result.success).toBe(true);
+      });
+
+      // Verify tool_choice modes are passed correctly to each provider
+      const results = parsed.results.results;
+
+      const autoResult = results.find(
+        (r: { provider: { label: string } }) => r.provider.label === 'Auto tool choice',
+      );
+      const autoOutput = JSON.parse(autoResult.response.output);
+      expect(autoOutput.tool_choice).toBe('auto');
+
+      const requiredResult = results.find(
+        (r: { provider: { label: string } }) => r.provider.label === 'Required tool choice',
+      );
+      const requiredOutput = JSON.parse(requiredResult.response.output);
+      expect(requiredOutput.tool_choice).toBe('required');
+
+      const specificResult = results.find(
+        (r: { provider: { label: string } }) => r.provider.label === 'Specific tool choice',
+      );
+      const specificOutput = JSON.parse(specificResult.response.output);
+      expect(specificOutput.tool_choice.type).toBe('function');
+      expect(specificOutput.tool_choice.function.name).toBe('get_weather');
+    });
+
+    it('3.3.2 - surfaces custom apiKeyEnvar in missing API key errors', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/openai-custom-api-key-envar.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'openai-custom-api-key-envar-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        env: {
+          OPENAI_API_KEY: '',
+          CUSTOM_ASSISTANT_KEY: '',
+        },
+      });
+
+      expect(exitCode).toBe(100);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const result = parsed.results.results[0];
+
+      expect(result.success).toBe(false);
+      expect(result.error).toContain(getOpenAiMissingApiKeyMessage('CUSTOM_ASSISTANT_KEY'));
+      expect(result.error).not.toContain(getOpenAiMissingApiKeyMessage());
+    });
+  });
+
+  describe('3.4 Python Providers', () => {
+    it('3.4.1 - Python provider with default call_api function', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/python-provider.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'python-provider-output.json');
+
+      // Run from the configs directory so relative paths work
+      const { exitCode, stdout, stderr } = runCli(
+        ['eval', '-c', configPath, '-o', outputPath, '--no-cache'],
+        { cwd: path.join(FIXTURES_DIR, 'configs') },
+      );
+
+      // Python provider requires Python to be installed
+      // If Python is not available, this test will fail gracefully
+      if (exitCode !== 0) {
+        const output = stdout + stderr;
+        if (output.includes('python') || output.includes('Python')) {
+          return;
+        }
+      }
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const result = parsed.results.results[0];
+
+      expect(result.response.output).toContain('Python Echo:');
+    });
+  });
+});
+
+describe('Data Loading Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('4.2 Tests Loading', () => {
+    it('4.2.2 - loads tests from CSV file', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/csv-tests.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'csv-tests-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: path.join(FIXTURES_DIR, 'configs'),
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have 2 test results (Alice and Bob from CSV)
+      expect(parsed.results.results.length).toBe(2);
+
+      // Verify the vars were loaded correctly
+      const outputs = parsed.results.results.map(
+        (r: { response: { output: string } }) => r.response.output,
+      );
+      expect(outputs.some((o: string) => o.includes('Alice'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Bob'))).toBe(true);
+    });
+
+    it('4.2.3 - loads tests from JSON file', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/json-tests.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'json-tests-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: path.join(FIXTURES_DIR, 'configs'),
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have 2 test results (Charlie and Diana from JSON)
+      expect(parsed.results.results.length).toBe(2);
+
+      const outputs = parsed.results.results.map(
+        (r: { response: { output: string } }) => r.response.output,
+      );
+      expect(outputs.some((o: string) => o.includes('Charlie'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Diana'))).toBe(true);
+    });
+  });
+
+  describe('4.3 Prompts Loading', () => {
+    it('4.3.2 - loads prompts from file:// reference', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/file-prompt.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'file-prompt-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: path.join(FIXTURES_DIR, 'configs'),
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const result = parsed.results.results[0];
+
+      // The prompt should have been loaded from the file and rendered
+      expect(result.response.output).toContain('bananas');
+    });
+  });
+});
+
+describe('Assertion Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('5.1 Built-in Assertions', () => {
+    it('5.1.3, 5.1.5, 5.1.6, 5.2.1 - various assertion types work correctly', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/assertions.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'assertions-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // All 6 tests should pass
+      expect(parsed.results.results.length).toBe(6);
+
+      for (const result of parsed.results.results) {
+        expect(result.success).toBe(true);
+      }
+    });
+
+    it('5.1.3 - equals assertion matches exactly', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/assertions.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'equals-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // First test uses equals assertion
+      const equalsResult = parsed.results.results[0];
+      expect(equalsResult.success).toBe(true);
+      expect(equalsResult.response.output).toBe('exact match test');
+    });
+
+    it('5.1.5 - regex assertion matches patterns', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/assertions.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'regex-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Second test uses regex assertion
+      const regexResult = parsed.results.results[1];
+      expect(regexResult.success).toBe(true);
+    });
+
+    it('5.1.6 - is-json assertion validates JSON', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/assertions.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'isjson-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Fourth test uses is-json assertion
+      const jsonResult = parsed.results.results[3];
+      expect(jsonResult.success).toBe(true);
+    });
+
+    it('5.2.1 - inline JavaScript assertion executes correctly', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/assertions.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'js-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Sixth test uses JavaScript assertion
+      const jsResult = parsed.results.results[5];
+      expect(jsResult.success).toBe(true);
+    });
+  });
+});

test/smoke/eval.test.ts

@@ -0,0 +1,211 @@
+/**
+ * Smoke tests for the eval command.
+ *
+ * These tests verify the core evaluation pipeline works correctly
+ * using the echo provider (no external API dependencies).
+ *
+ * @see docs/plans/smoke-tests.md for the full checklist
+ */
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+// Path to the built CLI binary
+const CLI_PATH = path.resolve(__dirname, '../../dist/src/main.js');
+const ROOT_DIR = path.resolve(__dirname, '../..');
+const FIXTURES_DIR = path.resolve(__dirname, 'fixtures');
+const OUTPUT_DIR = path.resolve(__dirname, '.temp-output');
+
+/**
+ * Helper to run the CLI and capture output
+ */
+function runCli(
+  args: string[],
+  options: { cwd?: string; expectError?: boolean; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [CLI_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 60000, // Eval can take longer
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+describe('Eval Smoke Tests', () => {
+  beforeAll(() => {
+    // Verify the built binary exists
+    if (!fs.existsSync(CLI_PATH)) {
+      throw new Error(`Built CLI not found at ${CLI_PATH}. Run 'npm run build' first.`);
+    }
+
+    // Create output directory for test artifacts
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    // Clean up output directory
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('1.4 Eval Command', () => {
+    it('1.4.1 - runs basic eval with echo provider', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const { stdout, exitCode } = runCli(['eval', '-c', configPath, '--no-cache']);
+
+      // Should complete successfully
+      expect(exitCode).toBe(0);
+
+      // Should show eval results
+      expect(stdout).toContain('PASS');
+    });
+
+    it('1.4.2 - outputs JSON format', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+      expect(fs.existsSync(outputPath)).toBe(true);
+
+      // Verify it's valid JSON with expected structure
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed).toHaveProperty('results');
+      expect(parsed.results).toHaveProperty('results');
+      expect(Array.isArray(parsed.results.results)).toBe(true);
+    });
+
+    it('1.4.3 - outputs YAML format', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'output.yaml');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+      expect(fs.existsSync(outputPath)).toBe(true);
+
+      // Verify it contains YAML-like content
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      expect(content).toContain('results:');
+    });
+
+    it('1.4.4 - outputs CSV format', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'output.csv');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+      expect(fs.existsSync(outputPath)).toBe(true);
+
+      // Verify it's CSV format (has header row with columns)
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const lines = content.trim().split('\n');
+      expect(lines.length).toBeGreaterThan(0);
+      // CSV should have comma-separated values
+      expect(lines[0]).toContain(',');
+    });
+
+    it('1.4.5 - respects --max-concurrency flag', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const { exitCode } = runCli([
+        'eval',
+        '-c',
+        configPath,
+        '--max-concurrency',
+        '1',
+        '--no-cache',
+      ]);
+
+      // Should complete successfully with concurrency limit
+      expect(exitCode).toBe(0);
+    });
+
+    it('1.4.6 - respects --repeat flag', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'repeat-output.json');
+
+      const { exitCode } = runCli([
+        'eval',
+        '-c',
+        configPath,
+        '--repeat',
+        '2',
+        '-o',
+        outputPath,
+        '--no-cache',
+      ]);
+
+      expect(exitCode).toBe(0);
+
+      // Verify we got repeated results
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      // With repeat=2 and 1 test case, we should have 2 results
+      expect(parsed.results.results.length).toBe(2);
+    });
+
+    it('1.4.7 - runs with --verbose flag', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const { stdout, exitCode } = runCli(['eval', '-c', configPath, '--verbose', '--no-cache']);
+
+      expect(exitCode).toBe(0);
+      // Verbose mode should produce more output
+      expect(stdout.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('1.7 Exit Codes', () => {
+    it('1.7.1 - returns exit code 0 when all assertions pass', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const { exitCode } = runCli(['eval', '-c', configPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+    });
+
+    it('1.7.2 - returns exit code 100 when assertions fail', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/failing-assertion.yaml');
+      const { exitCode } = runCli(['eval', '-c', configPath, '--no-cache']);
+
+      // Exit code 100 indicates test failures
+      expect(exitCode).toBe(100);
+    });
+
+    it('1.7.3 - returns exit code 1 for config errors', () => {
+      const { exitCode } = runCli(['eval', '-c', 'nonexistent-file.yaml', '--no-cache']);
+
+      expect(exitCode).toBe(1);
+    });
+  });
+
+  describe('Built-in Providers', () => {
+    it('3.1.1 - echo provider works correctly', () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/basic.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'echo-test.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache']);
+
+      expect(exitCode).toBe(0);
+
+      // Verify echo provider returns the prompt
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      const firstResult = parsed.results.results[0];
+
+      // Echo provider should return the prompt in the response
+      expect(firstResult.response.output).toContain('Hello');
+      expect(firstResult.response.output).toContain('World');
+    });
+  });
+});

test/smoke/export-import.test.ts

@@ -0,0 +1,129 @@
+/**
+ * Smoke coverage for portable eval export/import parity.
+ *
+ * The CLI smoke suite has a 30 second per-test budget, so this exercise keeps
+ * import and export in one process while still going through the real command
+ * actions, database migration path, and JSON output writer.
+ */
+import fs from 'fs';
+import path from 'path';
+
+import { Command } from 'commander';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { exportCommand } from '../../src/commands/export';
+import { importCommand } from '../../src/commands/import';
+import { closeDbIfOpen } from '../../src/database/index';
+import { runDbMigrations } from '../../src/migrate';
+
+const OUTPUT_DIR = path.resolve(__dirname, '.temp-output-export-import');
+const SAMPLE_EXPORT_PATH = path.resolve(__dirname, '../__fixtures__/sample-export.json');
+
+interface PortableEvalExport {
+  evalId: string;
+  config: Record<string, unknown>;
+  vars?: string[];
+  runtimeOptions?: Record<string, unknown>;
+  metadata: {
+    evaluationCreatedAt: string;
+  };
+  results: {
+    prompts: Array<{
+      id: string;
+      label: string;
+      provider: string;
+      raw: string;
+    }>;
+    results: Array<{
+      response?: { output?: unknown };
+      score: number;
+      success: boolean;
+      vars?: Record<string, unknown>;
+    }>;
+    stats: {
+      durationMs?: number;
+      evaluationDurationMs?: number;
+      errors: number;
+      failures: number;
+      generationDurationMs?: number;
+      successes: number;
+    };
+  };
+}
+
+function parityFields(output: PortableEvalExport) {
+  return {
+    evalId: output.evalId,
+    config: output.config,
+    vars: output.vars,
+    runtimeOptions: output.runtimeOptions,
+    evaluationCreatedAt: output.metadata.evaluationCreatedAt,
+    prompts: output.results.prompts.map(({ id, label, provider, raw }) => ({
+      id,
+      label,
+      provider,
+      raw,
+    })),
+    results: output.results.results.map(({ response, score, success, vars }) => ({
+      output: response?.output,
+      score,
+      success,
+      vars,
+    })),
+    stats: {
+      durationMs: output.results.stats.durationMs,
+      evaluationDurationMs: output.results.stats.evaluationDurationMs,
+      errors: output.results.stats.errors,
+      failures: output.results.stats.failures,
+      generationDurationMs: output.results.stats.generationDurationMs,
+      successes: output.results.stats.successes,
+    },
+  };
+}
+
+describe('Export/import smoke tests', () => {
+  beforeAll(async () => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+    await runDbMigrations();
+  });
+
+  afterAll(async () => {
+    await closeDbIfOpen();
+    fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    process.exitCode = undefined;
+  });
+
+  it('round-trips a portable eval through import and export', async () => {
+    process.exitCode = undefined;
+
+    const importPath = path.join(OUTPUT_DIR, 'portable-eval.json');
+    const reexportPath = path.join(OUTPUT_DIR, 'reexported-eval.json');
+    const exportData = JSON.parse(
+      fs.readFileSync(SAMPLE_EXPORT_PATH, 'utf-8'),
+    ) as PortableEvalExport;
+    exportData.evalId = 'eval-smoke-export-import-cycle';
+    exportData.vars = ['riddle', 'unused_export_var'];
+    exportData.runtimeOptions = { cache: false, maxConcurrency: 2, repeat: 3 };
+    fs.writeFileSync(importPath, JSON.stringify(exportData));
+
+    const importProgram = new Command();
+    importCommand(importProgram);
+    await importProgram.parseAsync(['node', 'smoke', 'import', importPath]);
+    expect(process.exitCode).toBeUndefined();
+
+    const exportProgram = new Command();
+    exportCommand(exportProgram);
+    await exportProgram.parseAsync([
+      'node',
+      'smoke',
+      'export',
+      'eval',
+      exportData.evalId,
+      '--output',
+      reexportPath,
+    ]);
+    expect(process.exitCode).toBeUndefined();
+
+    const reexportData = JSON.parse(fs.readFileSync(reexportPath, 'utf-8')) as PortableEvalExport;
+    expect(parityFields(reexportData)).toEqual(parityFields(exportData));
+  });
+});

test/smoke/extension-hooks.test.ts

@@ -0,0 +1,121 @@
+/**
+ * Smoke tests for extension hooks.
+ *
+ * Verifies that a Python extension hook returning its context does not break
+ * function-based prompts — the hook's subprocess JSON round-trip used to drop
+ * the non-serializable prompt function, sending raw Python source to the
+ * provider instead (regression test for
+ * https://github.com/promptfoo/promptfoo/issues/9653).
+ */
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+// Path to the built CLI binary
+const CLI_PATH = path.resolve(__dirname, '../../dist/src/main.js');
+const ROOT_DIR = path.resolve(__dirname, '../..');
+const FIXTURES_DIR = path.resolve(__dirname, 'fixtures');
+const OUTPUT_DIR = path.resolve(__dirname, '.temp-output-extension-hooks');
+
+/**
+ * Helper to run the CLI and capture output
+ */
+function runCli(
+  args: string[],
+  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [CLI_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 60000,
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+function findPythonPath(): string | undefined {
+  const candidates: Array<[string, string[]]> = [];
+  if (process.env.PROMPTFOO_PYTHON) {
+    candidates.push([process.env.PROMPTFOO_PYTHON, ['-c', 'import sys; print(sys.executable)']]);
+  }
+  if (process.platform === 'win32') {
+    candidates.push(['py', ['-3', '-c', 'import sys; print(sys.executable)']]);
+  }
+  candidates.push(
+    ['python3', ['-c', 'import sys; print(sys.executable)']],
+    ['python', ['-c', 'import sys; print(sys.executable)']],
+  );
+
+  for (const [command, args] of candidates) {
+    const result = spawnSync(command, args, { encoding: 'utf-8', timeout: 5000 });
+    if (result.status === 0 && result.stdout.trim()) {
+      return result.stdout.trim();
+    }
+  }
+  return undefined;
+}
+
+const PYTHON_PATH = findPythonPath();
+
+describe('Extension Hook Smoke Tests', () => {
+  beforeAll(() => {
+    if (!fs.existsSync(CLI_PATH)) {
+      throw new Error(`Built CLI not found at ${CLI_PATH}. Run 'npm run build' first.`);
+    }
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+  });
+
+  it.skipIf(!PYTHON_PATH)(
+    'executes a Python prompt function when a Python extension hook returns its context (issue #9653)',
+    () => {
+      const configPath = path.join(FIXTURES_DIR, 'configs/python-prompt-with-extension.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'python-prompt-with-extension-output.json');
+
+      const { exitCode } = runCli(
+        [
+          'eval',
+          '-c',
+          configPath,
+          '-o',
+          outputPath,
+          '--no-cache',
+          '--no-share',
+          '--no-table',
+          '--no-progress-bar',
+        ],
+        {
+          cwd: path.join(FIXTURES_DIR, 'configs'),
+          env: {
+            PROMPTFOO_CONFIG_DIR: OUTPUT_DIR,
+            PROMPTFOO_PYTHON: PYTHON_PATH,
+            PROMPTFOO_DISABLE_SHARING: 'true',
+            PROMPTFOO_DISABLE_TELEMETRY: 'true',
+            PROMPTFOO_DISABLE_UPDATE: 'true',
+          },
+        },
+      );
+
+      expect(exitCode).toBe(0);
+
+      const parsed = JSON.parse(fs.readFileSync(outputPath, 'utf-8'));
+      const result = parsed.results.results[0];
+
+      // The executed prompt function produces chat messages. If the hook's JSON
+      // round-trip had dropped the function, the raw Python source would have
+      // been rendered into the prompt instead.
+      expect(result.prompt.raw).toContain('What is Linear Algebra?');
+      expect(result.prompt.raw).not.toContain('def create_prompt');
+    },
+  );
+});

test/smoke/features-and-assertions.test.ts

@@ -0,0 +1,309 @@
+/**
+ * Smoke tests for additional assertions, transforms, and feature integration.
+ *
+ * Tests provider named functions, test generators, additional assertion types,
+ * response transforms, and advanced config features.
+ *
+ * @see docs/plans/smoke-tests.md for the full checklist
+ */
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+// Path to the built CLI binary
+const CLI_PATH = path.resolve(__dirname, '../../dist/src/main.js');
+const ROOT_DIR = path.resolve(__dirname, '../..');
+const FIXTURES_DIR = path.resolve(__dirname, 'fixtures');
+const CONFIGS_DIR = path.resolve(FIXTURES_DIR, 'configs');
+const OUTPUT_DIR = path.resolve(__dirname, '.temp-output-features');
+
+/**
+ * Helper to run the CLI and capture output
+ */
+function runCli(
+  args: string[],
+  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
+): { stdout: string; stderr: string; exitCode: number } {
+  const result = spawnSync('node', [CLI_PATH, ...args], {
+    cwd: options.cwd || ROOT_DIR,
+    encoding: 'utf-8',
+    env: { ...process.env, ...options.env, NO_COLOR: '1' },
+    timeout: 60000,
+  });
+
+  return {
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    exitCode: result.status ?? 1,
+  };
+}
+
+describe('Named Function Provider Smoke Tests', () => {
+  beforeAll(() => {
+    if (!fs.existsSync(CLI_PATH)) {
+      throw new Error(`Built CLI not found at ${CLI_PATH}. Run 'npm run build' first.`);
+    }
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('5.1 Additional Assertions', () => {
+    it('5.1.2 - not-contains assertion', () => {
+      const configPath = path.join(CONFIGS_DIR, 'not-contains-assertion.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'not-contains-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+  });
+
+  describe('3.4 Python Provider Named Functions', () => {
+    it('3.4.2 - Python provider with named function', () => {
+      const configPath = path.join(CONFIGS_DIR, 'provider-python-named.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'python-named-output.json');
+
+      const { exitCode, stdout, stderr } = runCli(
+        ['eval', '-c', configPath, '-o', outputPath, '--no-cache'],
+        { cwd: CONFIGS_DIR },
+      );
+
+      // Python may not be available in all environments
+      if (exitCode !== 0) {
+        const output = stdout + stderr;
+        if (
+          output.toLowerCase().includes('python') &&
+          (output.toLowerCase().includes('not found') ||
+            output.toLowerCase().includes('no such file'))
+        ) {
+          return;
+        }
+      }
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+      expect(parsed.results.results[0].response.output).toContain('Python Custom Echo:');
+    });
+  });
+});
+
+describe('Test Generator Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('4.2 JavaScript Test Generator', () => {
+    it('4.2.7 - loads tests from JavaScript generator file', () => {
+      const configPath = path.join(CONFIGS_DIR, 'js-test-generator.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'js-generator-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have 3 test results from the generator
+      expect(parsed.results.results.length).toBe(3);
+
+      const outputs = parsed.results.results.map(
+        (r: { response: { output: string } }) => r.response.output,
+      );
+      expect(outputs.some((o: string) => o.includes('Generated1'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Generated2'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Generated3'))).toBe(true);
+    });
+  });
+});
+
+describe('Additional Assertion Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('5.1 Built-in Assertions', () => {
+    it('5.1.1 - contains assertion', () => {
+      const configPath = path.join(CONFIGS_DIR, 'contains-assertion.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'contains-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+
+    it('5.1.4 - starts-with assertion', () => {
+      const configPath = path.join(CONFIGS_DIR, 'starts-with-assertion.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'starts-with-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+
+    it('5.1.7 - contains-json assertion', () => {
+      const configPath = path.join(CONFIGS_DIR, 'contains-json-assertion.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'contains-json-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+  });
+});
+
+describe('Transform Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('6.1 Response Transforms', () => {
+    it('6.1.1 - transform response expression', () => {
+      const configPath = path.join(CONFIGS_DIR, 'transform-response.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'transform-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+    });
+  });
+});
+
+describe('Feature Integration Smoke Tests', () => {
+  beforeAll(() => {
+    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(OUTPUT_DIR)) {
+      fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
+    }
+  });
+
+  describe('7.1 Provider Config Options', () => {
+    it('7.1.1 - provider with config options', () => {
+      const configPath = path.join(CONFIGS_DIR, 'provider-with-config.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'provider-config-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.results.results[0].success).toBe(true);
+      // Check that the provider label is used
+      expect(parsed.results.prompts[0].provider).toBe('Custom Echo Provider');
+    });
+  });
+
+  describe('7.2 DefaultTest', () => {
+    it('7.2.1 - defaultTest applies assertions to all tests', () => {
+      const configPath = path.join(CONFIGS_DIR, 'default-test.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'default-test-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // All 3 tests should pass with the defaultTest assertion
+      expect(parsed.results.results.length).toBe(3);
+      expect(parsed.results.results.every((r: { success: boolean }) => r.success)).toBe(true);
+    });
+  });
+
+  describe('7.3 Scenarios', () => {
+    it('7.3.1 - scenarios feature loads and runs correctly', () => {
+      const configPath = path.join(CONFIGS_DIR, 'scenarios.yaml');
+      const outputPath = path.join(OUTPUT_DIR, 'scenarios-output.json');
+
+      const { exitCode } = runCli(['eval', '-c', configPath, '-o', outputPath, '--no-cache'], {
+        cwd: CONFIGS_DIR,
+      });
+
+      expect(exitCode).toBe(0);
+
+      const content = fs.readFileSync(outputPath, 'utf-8');
+      const parsed = JSON.parse(content);
+
+      // Should have 3 tests total (2 from US scenario + 1 from EU scenario)
+      expect(parsed.results.results.length).toBe(3);
+      expect(parsed.results.results.every((r: { success: boolean }) => r.success)).toBe(true);
+
+      // Verify scenarios data
+      const outputs = parsed.results.results.map(
+        (r: { response: { output: string } }) => r.response.output,
+      );
+      expect(outputs.some((o: string) => o.includes('New York'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('California'))).toBe(true);
+      expect(outputs.some((o: string) => o.includes('Paris'))).toBe(true);
+    });
+  });
+});

test/smoke/fixtures/assertions/check-length.js

@@ -0,0 +1,22 @@
+/**
+ * JavaScript assertion file (5.2.2)
+ * Checks that output meets minimum length requirement
+ */
+module.exports = (output, context) => {
+  const minLength = context.test?.assert?.[0]?.config?.minLength || 5;
+  const actualLength = output.length;
+
+  if (actualLength >= minLength) {
+    return {
+      pass: true,
+      score: 1.0,
+      reason: `Output length (${actualLength}) meets minimum (${minLength})`,
+    };
+  }
+
+  return {
+    pass: false,
+    score: actualLength / minLength,
+    reason: `Output length (${actualLength}) below minimum (${minLength})`,
+  };
+};

test/smoke/fixtures/assertions/check_keywords.py

@@ -0,0 +1,22 @@
+"""
+Python assertion file (5.2.5)
+Checks that output contains expected keywords
+"""
+
+
+def get_assert(output, context):
+    """Check if output contains the expected keyword."""
+    expected = context.get("test", {}).get("vars", {}).get("expected_word", "hello")
+
+    if expected.lower() in output.lower():
+        return {
+            "pass": True,
+            "score": 1.0,
+            "reason": f"Output contains '{expected}'",
+        }
+
+    return {
+        "pass": False,
+        "score": 0.0,
+        "reason": f"Output does not contain '{expected}'",
+    }

test/smoke/fixtures/assertions/dynamic-value.js

@@ -0,0 +1,11 @@
+/**
+ * Dynamic assertion value script (#6253)
+ *
+ * This script returns a value to be used in an assertion.
+ * Tests that file:// references in assertion values use script output.
+ */
+
+module.exports = function () {
+  // Return a dynamic value that the assertion will use
+  return 'DynamicValue';
+};

test/smoke/fixtures/assertions/dynamic-value.py

@@ -0,0 +1,10 @@
+"""
+Dynamic assertion value script (#6253)
+
+This script returns a value to be used in an assertion.
+Tests that file:// references in assertion values use script output.
+"""
+
+
+def get_value():
+    return "PythonDynamicValue"

test/smoke/fixtures/configs/assert-dynamic-var.js

@@ -0,0 +1,31 @@
+// Assertion function that verifies dynamic vars are resolved
+// Tests fix for GitHub issue #7334
+module.exports = function (_output, context) {
+  const dynamicVar = context.vars.DYNAMIC_VAR;
+
+  // Check if the variable was resolved (should be an ISO date string)
+  // or still has the file:// prefix (bug)
+  if (typeof dynamicVar === 'string' && dynamicVar.startsWith('file://')) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `BUG: DYNAMIC_VAR was not resolved! Got raw file path: ${dynamicVar}`,
+    };
+  }
+
+  // Validate it's a valid ISO date
+  const date = new Date(dynamicVar);
+  if (isNaN(date.getTime())) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `DYNAMIC_VAR is not a valid date: ${dynamicVar}`,
+    };
+  }
+
+  return {
+    pass: true,
+    score: 1,
+    reason: `DYNAMIC_VAR was correctly resolved to: ${dynamicVar}`,
+  };
+};

test/smoke/fixtures/configs/assertions.yaml

@@ -0,0 +1,50 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - various assertion types'
+
+providers:
+  - echo
+
+prompts:
+  - '{{input}}'
+
+tests:
+  # Test equals assertion
+  - vars:
+      input: 'exact match test'
+    assert:
+      - type: equals
+        value: 'exact match test'
+
+  # Test regex assertion
+  - vars:
+      input: 'The answer is 42'
+    assert:
+      - type: regex
+        value: 'answer.*\d+'
+
+  # Test starts-with assertion
+  - vars:
+      input: 'Hello World'
+    assert:
+      - type: starts-with
+        value: 'Hello'
+
+  # Test is-json assertion
+  - vars:
+      input: '{"key": "value", "number": 123}'
+    assert:
+      - type: is-json
+
+  # Test not-contains assertion
+  - vars:
+      input: 'This is safe content'
+    assert:
+      - type: not-contains
+        value: 'dangerous'
+
+  # Test inline JavaScript assertion
+  - vars:
+      input: 'count to five: 1 2 3 4 5'
+    assert:
+      - type: javascript
+        value: output.includes('5') && output.length > 10

test/smoke/fixtures/configs/basic.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://promptfoo.dev/config-schema.json",
+  "description": "Smoke test - JSON config format",
+  "providers": ["echo"],
+  "prompts": ["Hello {{name}}"],
+  "tests": [
+    {
+      "vars": { "name": "JSON" },
+      "assert": [{ "type": "contains", "value": "Hello" }, { "type": "contains", "value": "JSON" }]
+    }
+  ]
+}

test/smoke/fixtures/configs/basic.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - basic config validation'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: Hello
+      - type: contains
+        value: World

test/smoke/fixtures/configs/circular-ref-7266.yaml

@@ -0,0 +1,23 @@
+# Test config for GitHub issue #7266
+# Provider returns data with circular references (simulating leaked Timeout objects)
+# This should fail on main but pass with the fix
+#
+# NOTE: This test intentionally uses a custom provider instead of 'echo' because:
+# - The echo provider cannot generate circular references in its output
+# - This test must verify that circular reference objects (like Node.js Timeout)
+#   are properly sanitized before database storage
+# - The custom provider creates a controlled circular reference structure
+#   (similar to _idlePrev/_idleNext in Node.js timers) to reproduce the bug
+description: 'Regression test for #7266 - circular reference in provider response'
+
+providers:
+  - file://../providers/circular-ref-provider.js
+
+prompts:
+  - 'Test prompt'
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: 'Processed'

test/smoke/fixtures/configs/cjs-module-exports.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.1.1: CJS provider with module.exports still works after ESM migration
+# Bug #6501: .js files with CJS syntax failed to load in 0.120.0
+description: 'Regression test - CJS module.exports provider'
+
+providers:
+  - file://../providers/cjs-module-exports.js
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: CjsTest
+    assert:
+      - type: contains
+        value: 'CJS Echo:'
+      - type: contains
+        value: CjsTest

test/smoke/fixtures/configs/cjs-with-require.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.1.2: CJS provider that uses require() internally
+# Bug #6468: require() resolution was broken in 0.120.0
+description: 'Regression test - CJS provider with require()'
+
+providers:
+  - file://../providers/cjs-with-require.js
+
+prompts:
+  - 'Test {{value}}'
+
+tests:
+  - vars:
+      value: RequireTest
+    assert:
+      - type: contains
+        value: 'Require Test:'
+      - type: contains
+        value: 'platform='
+      - type: contains
+        value: RequireTest

test/smoke/fixtures/configs/class-provider-7353.yaml

@@ -0,0 +1,16 @@
+# Test config for #7353 - class-based provider with prototype id() method
+# This verifies that class-based providers work correctly through the CLI.
+description: 'Test class-based provider with prototype id()'
+
+providers:
+  - file://../providers/class-provider-prototype-id.js
+
+prompts:
+  - 'Hello World'
+
+tests:
+  - assert:
+      - type: contains
+        value: ClassProvider
+      - type: contains
+        value: Hello World

test/smoke/fixtures/configs/config-base.yaml

@@ -0,0 +1,9 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Base config for testing multiple config merging
+description: 'Base config'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'

test/smoke/fixtures/configs/config-cjs.cjs

@@ -0,0 +1,17 @@
+/**
+ * CJS config using module.exports object (2.3.1)
+ */
+module.exports = {
+  description: 'Smoke test - CJS config format',
+  providers: ['echo'],
+  prompts: ['Hello from CJS config: {{name}}'],
+  tests: [
+    {
+      vars: { name: 'CommonJS' },
+      assert: [
+        { type: 'contains', value: 'Hello from CJS' },
+        { type: 'contains', value: 'CommonJS' },
+      ],
+    },
+  ],
+};

test/smoke/fixtures/configs/config-esm.mjs

@@ -0,0 +1,17 @@
+/**
+ * ESM config using export default (2.3.5)
+ */
+export default {
+  description: 'Smoke test - ESM config format',
+  providers: ['echo'],
+  prompts: ['Hello from ESM config: {{name}}'],
+  tests: [
+    {
+      vars: { name: 'ESModule' },
+      assert: [
+        { type: 'contains', value: 'Hello from ESM' },
+        { type: 'contains', value: 'ESModule' },
+      ],
+    },
+  ],
+};

test/smoke/fixtures/configs/config-tests.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Complete config for testing config-based features
+description: 'Smoke test - config extension'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: ConfigMergeTest
+    assert:
+      - type: contains
+        value: ConfigMergeTest

test/smoke/fixtures/configs/config-ts.ts

@@ -0,0 +1,34 @@
+/**
+ * TypeScript config using export default (2.4.1)
+ *
+ * Uses inline type to avoid depcheck issues with 'promptfoo' import.
+ * In real usage, you would import from 'promptfoo':
+ * import type { UnifiedConfig } from 'promptfoo';
+ */
+
+interface UnifiedConfig {
+  description?: string;
+  providers: string[];
+  prompts: string[];
+  tests: Array<{
+    vars: Record<string, string>;
+    assert: Array<{ type: string; value: string }>;
+  }>;
+}
+
+const config: UnifiedConfig = {
+  description: 'Smoke test - TypeScript config format',
+  providers: ['echo'],
+  prompts: ['Hello from TypeScript config: {{name}}'],
+  tests: [
+    {
+      vars: { name: 'TypeScript' },
+      assert: [
+        { type: 'contains', value: 'Hello from TypeScript' },
+        { type: 'contains', value: 'TypeScript' },
+      ],
+    },
+  ],
+};
+
+export default config;

test/smoke/fixtures/configs/contains-all-assertion.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing contains-all assertion
+description: 'Smoke test - contains-all assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello World, this is a test message'
+
+tests:
+  - assert:
+      # All values must be present
+      - type: contains-all
+        value:
+          - Hello
+          - World
+          - test

test/smoke/fixtures/configs/contains-any-assertion.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing contains-any assertion
+description: 'Smoke test - contains-any assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'The quick brown fox'
+
+tests:
+  - assert:
+      # At least one value must be present
+      - type: contains-any
+        value:
+          - elephant
+          - fox
+          - giraffe

test/smoke/fixtures/configs/contains-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - contains assertion (5.1.1)'
+
+providers:
+  - echo
+
+prompts:
+  - 'The quick brown fox jumps over the lazy dog'
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: 'quick brown'
+      - type: contains
+        value: 'lazy dog'

test/smoke/fixtures/configs/contains-json-assertion.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - contains-json assertion (5.1.7)'
+
+providers:
+  - echo
+
+prompts:
+  # The echo provider echoes the prompt, so include JSON in the prompt
+  - 'Here is the data: {"status": "success", "code": 200, "message": "Hello"}'
+
+tests:
+  - vars: {}
+    assert:
+      # contains-json without value just checks that valid JSON is present in output
+      - type: contains-json
+      # With a value, it validates against a JSON schema
+      - type: contains-json
+        value:
+          type: object
+          required:
+            - status
+            - code
+          properties:
+            status:
+              type: string
+            code:
+              type: number

test/smoke/fixtures/configs/contains-quoted-commas.yaml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing quoted comma parsing across all contains assertion types
+description: 'Smoke test - quoted comma parsing in contains assertions'
+
+providers:
+  - echo
+
+prompts:
+  - 'hello, world and foo bar'
+
+tests:
+  # contains-any with quoted comma value (string format)
+  - assert:
+      - type: contains-any
+        value: '"hello, world",universe'
+
+  # contains-all with quoted comma values (string format)
+  - assert:
+      - type: contains-all
+        value: '"hello, world","foo"'
+
+  # icontains-any with quoted comma value
+  - assert:
+      - type: icontains-any
+        value: '"HELLO, WORLD",universe'
+
+  # icontains-all with quoted comma values
+  - assert:
+      - type: icontains-all
+        value: '"HELLO, WORLD","FOO"'

test/smoke/fixtures/configs/conversation-relevance-ssti.yaml

@@ -0,0 +1,23 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - conversation relevance SSTI'
+
+prompts:
+  - 'unused prompt'
+
+providers:
+  - echo
+
+defaultTest:
+  options:
+    provider: 'exec:node ../scripts/capture-conversation-relevance-grader.js'
+
+tests:
+  - vars:
+      country: France
+      capital: Paris
+      _conversation:
+        - input: 'What is the capital of {{country}}?'
+          output: 'The answer is {{capital}} and {{ env.OPENAI_API_KEY }} and {% for x in range(3) %}{{x}}{% endfor %}'
+    assert:
+      - type: conversation-relevance
+        threshold: 0

test/smoke/fixtures/configs/cost-assertion.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing cost assertion
+description: 'Smoke test - cost assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test prompt for cost check'
+
+tests:
+  - assert:
+      # Echo provider has 0 cost, should pass with any threshold
+      - type: cost
+        threshold: 1.0

test/smoke/fixtures/configs/csv-tests.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - CSV test loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, you are {{age}} years old'
+
+tests: file://tests.csv

test/smoke/fixtures/configs/default-test.yaml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - defaultTest feature (7.2.1)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+# Default assertions applied to all tests
+defaultTest:
+  assert:
+    - type: contains
+      value: 'Hello'
+
+tests:
+  - vars:
+      name: Alice
+  - vars:
+      name: Bob
+  - vars:
+      name: Charlie

test/smoke/fixtures/configs/defaulttest-llm-rubric-vars-default.yaml

@@ -0,0 +1,7 @@
+options:
+  provider:
+    id: file://../providers/defaulttest-llm-rubric-grader.cjs
+
+assert:
+  - type: llm-rubric
+    value: 'Does the output correctly reference the input: {{myVar}}?'

test/smoke/fixtures/configs/defaulttest-llm-rubric-vars-tests.yaml

@@ -0,0 +1,4 @@
+- vars:
+    myVar: 'hello world'
+- vars:
+    myVar: 'goodbye moon'

test/smoke/fixtures/configs/defaulttest-llm-rubric-vars.yaml

@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Regression test - defaultTest llm-rubric vars'
+
+prompts:
+  - 'static model output'
+
+providers:
+  - echo
+
+defaultTest: file://defaulttest-llm-rubric-vars-default.yaml
+
+tests:
+  - file://defaulttest-llm-rubric-vars-tests.yaml

test/smoke/fixtures/configs/delay-test.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Simple config for testing --delay flag
+description: 'Smoke test - delay between tests'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{num}}'
+
+tests:
+  - vars:
+      num: '1'
+  - vars:
+      num: '2'
+  - vars:
+      num: '3'

test/smoke/fixtures/configs/dynamic-var-assertion-7334.yaml

@@ -0,0 +1,23 @@
+# Regression test for GitHub Issue #7334
+# Dynamic variables should be resolved when passed to assertion functions
+# https://github.com/promptfoo/promptfoo/issues/7334
+
+providers:
+  - echo
+
+prompts:
+  - 'The dynamic variable is: {{DYNAMIC_VAR}}'
+
+defaultTest:
+  vars:
+    # This dynamic variable is loaded from a JavaScript file
+    DYNAMIC_VAR: file://dynamic-var-generator.js
+
+tests:
+  - description: 'Dynamic vars should be resolved in assertion context.vars'
+    assert:
+      # This assertion function accesses context.vars.DYNAMIC_VAR
+      # Before fix: received "file://dynamic-var-generator.js"
+      # After fix: receives the resolved ISO date string
+      - type: javascript
+        value: file://assert-dynamic-var.js

test/smoke/fixtures/configs/dynamic-var-generator.js

@@ -0,0 +1,7 @@
+// Dynamic variable that returns current ISO UTC timestamp
+// Used to test that file:// vars are resolved before being passed to assertions
+module.exports = function (_varName, _prompt, _vars, _provider) {
+  return {
+    output: new Date().toISOString(),
+  };
+};

test/smoke/fixtures/configs/empty-vars.yaml

@@ -0,0 +1,10 @@
+description: 'Test empty vars handling'
+providers:
+  - echo
+prompts:
+  - 'Static prompt'
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: Static

test/smoke/fixtures/configs/ends-with-assertion.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing regex assertion with end-of-string pattern
+description: 'Smoke test - regex ends-with pattern'
+
+providers:
+  - echo
+
+prompts:
+  - 'The answer is 42.'
+
+tests:
+  - assert:
+      # Regex pattern to match string ending with "42."
+      - type: regex
+        value: '42\.$'
+      # Regex pattern to match string ending with "."
+      - type: regex
+        value: '\.$'

test/smoke/fixtures/configs/env-var-test.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing --env-file flag
+description: 'Smoke test - environment variable loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'API Key: {{api_key}}, Secret: {{secret}}'
+
+tests:
+  - vars:
+      api_key: '{{env.TEST_API_KEY}}'
+      secret: '{{env.TEST_SECRET}}'
+    assert:
+      - type: contains
+        value: 'API Key:'

test/smoke/fixtures/configs/exec-provider-stdin.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - exec provider closes stdin (regression for #8686)'
+
+providers:
+  - 'exec:node ../providers/exec-provider-reads-stdin.js'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: stdin closed

test/smoke/fixtures/configs/exec-provider.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - exec provider'
+
+providers:
+  - 'exec:echo "Echo from exec: {{prompt}}"'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: Echo from exec

test/smoke/fixtures/configs/failing-assertion.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - config with failing assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      # This assertion will fail because echo returns "Hello World"
+      # but we're asserting it contains "IMPOSSIBLE_STRING_NOT_IN_OUTPUT"
+      - type: contains
+        value: IMPOSSIBLE_STRING_NOT_IN_OUTPUT_12345

test/smoke/fixtures/configs/failing-tests.yaml

@@ -0,0 +1,38 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with some intentionally failing tests for --filter-failing testing
+description: 'Smoke test - config with failing tests'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - description: 'passing test 1'
+    vars:
+      name: Alice
+    assert:
+      - type: contains
+        value: Alice
+
+  - description: 'failing test 1'
+    vars:
+      name: Bob
+    assert:
+      - type: contains
+        value: 'WILL_NOT_MATCH_12345'
+
+  - description: 'passing test 2'
+    vars:
+      name: Charlie
+    assert:
+      - type: contains
+        value: Charlie
+
+  - description: 'failing test 2'
+    vars:
+      name: Diana
+    assert:
+      - type: contains
+        value: 'ALSO_NOT_MATCHING_67890'

test/smoke/fixtures/configs/file-prompt.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - file:// prompt loading'
+
+providers:
+  - echo
+
+prompts:
+  - file://prompt-template.txt
+
+tests:
+  - vars:
+      topic: bananas
+    assert:
+      - type: contains
+        value: bananas

test/smoke/fixtures/configs/file-provider-env-7079.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - file provider env placeholders'
+
+providers:
+  - file://../providers/file-provider-env-7079.yaml
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: World

test/smoke/fixtures/configs/file-ref-assertion-value.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test for #6253: file:// references in assertion values use script output
+description: 'Regression test - file:// references in assertion values'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test DynamicValue here'
+
+tests:
+  - assert:
+      # The file:// reference should execute the script and use its return value
+      - type: contains
+        value: file://../assertions/dynamic-value.js

test/smoke/fixtures/configs/file-ref-vars.yaml

@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test for #6393: file:// references in vars context for runtime loading
+description: 'Regression test - file:// references in vars'
+
+providers:
+  - echo
+
+prompts:
+  - 'Value is {{dynamicVar}}'
+
+# Load tests from a file that includes vars
+tests: file://../data/tests-with-vars.yaml

test/smoke/fixtures/configs/function-provider-defaulttest.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test for #6174: function providers in defaultTest.options.provider
+description: 'Regression test - function providers in defaultTest'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{name}}'
+
+# Function provider in defaultTest.options.provider should work
+defaultTest:
+  assert:
+    - type: contains
+      value: Test
+
+tests:
+  - vars:
+      name: FunctionProvider

test/smoke/fixtures/configs/function-providers-9383.ts

@@ -0,0 +1,35 @@
+/**
+ * Regression fixture for #9383: multiple CallApiFunction providers must each
+ * survive combineConfigs (the JSON.stringify-based dedupe used to collapse them
+ * into a single entry).
+ *
+ * Mirrors the repro from the issue: three labeled function providers that each
+ * embed their label in the response so the eval output is distinguishable.
+ */
+
+interface CallApiFunction {
+  (prompt: string): Promise<{ output: string }>;
+  label?: string;
+}
+
+interface UnifiedConfig {
+  description?: string;
+  providers: CallApiFunction[];
+  prompts: string[];
+  tests: Array<{ vars: Record<string, string> }>;
+}
+
+const makeProvider = (label: string): CallApiFunction => {
+  const fn: CallApiFunction = async (prompt) => ({ output: `${label}: ${prompt}` });
+  fn.label = label;
+  return fn;
+};
+
+const config: UnifiedConfig = {
+  description: 'Smoke test - multiple function providers (#9383)',
+  providers: [makeProvider('a'), makeProvider('b'), makeProvider('c')],
+  prompts: ['{{prompt}}'],
+  tests: [{ vars: { prompt: 'hi' } }],
+};
+
+export default config;

test/smoke/fixtures/configs/glob-prompts.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing multiple prompts from files
+description: 'Smoke test - multiple file prompts'
+
+providers:
+  - echo
+
+prompts:
+  - file://../prompts/greeting.txt
+  - file://../prompts/farewell.txt
+
+tests:
+  - vars:
+      name: GlobTest
+    assert:
+      - type: contains
+        value: GlobTest

test/smoke/fixtures/configs/go-provider.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.5.1: Go provider works after ESM migration
+# Bug #6506: Go provider wrapper failed in 0.120.0
+description: 'Regression test - Go provider'
+
+providers:
+  - file://../providers/echo-go.go
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: GoTest
+    assert:
+      - type: contains
+        value: 'Go Echo:'
+      - type: contains
+        value: GoTest

test/smoke/fixtures/configs/icontains-assertion.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing icontains (case-insensitive contains) assertion
+description: 'Smoke test - icontains assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Response: HELLO World Test'
+
+tests:
+  - assert:
+      # Case-insensitive match - "hello" matches "HELLO"
+      - type: icontains
+        value: hello
+      # Case-insensitive match - "WORLD" matches "World"
+      - type: icontains
+        value: WORLD
+      # Case-insensitive match - mixed case
+      - type: icontains
+        value: wOrLd

test/smoke/fixtures/configs/inline-js-assertion.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing inline JavaScript assertion
+description: 'Smoke test - inline JavaScript assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'The answer is 42'
+
+tests:
+  - assert:
+      # Inline JavaScript expression
+      - type: javascript
+        value: 'output.includes("42") && output.length > 10'

test/smoke/fixtures/configs/inline-js-process.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.1.3: Inline JS with process.mainModule shim
+# Bug #6606: Inline transforms using process.mainModule.require broke in 0.120.0
+#
+# After ESM migration, process.mainModule is undefined. The fix adds a shim
+# that provides process.mainModule.require via createRequire().
+description: 'Regression test - inline JS with process access'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{value}}'
+
+tests:
+  - vars:
+      value: ProcessTest
+    assert:
+      - type: javascript
+        value: |
+          // Test that process object is accessible in inline JS
+          // This broke in 0.120.0 because process.mainModule was undefined
+          // Note: 'output' is already defined in the context, use it directly
+          const hasProcess = typeof process !== 'undefined';
+          const hasEnv = hasProcess && typeof process.env !== 'undefined';
+          // The assertion should pass if process is accessible
+          return output.includes('ProcessTest') && hasProcess && hasEnv;

test/smoke/fixtures/configs/invalid.yaml

@@ -0,0 +1,14 @@
+# This config is intentionally invalid for testing validation errors
+description: 'Invalid config for smoke testing'
+
+# Missing required 'providers' field
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: invalid-assertion-type-that-does-not-exist
+        value: test

test/smoke/fixtures/configs/js-assertion.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JavaScript file assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'This is a test message with some content: {{topic}}'
+
+tests:
+  - vars:
+      topic: bananas
+    assert:
+      - type: javascript
+        value: file://../assertions/check-length.js
+        config:
+          minLength: 10

test/smoke/fixtures/configs/js-test-generator.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JavaScript test generator (4.2.7)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests: file://../data/test-generator.js

test/smoke/fixtures/configs/json-chat-format.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 11.2.1: JSON chat message parsing
+# Bug #6568: Incorrect parsing of JSON vs non-JSON chat messages in 0.120.1
+description: 'Regression test - JSON chat message parsing'
+
+providers:
+  - echo
+
+# This uses a JSON file containing chat messages in array format
+prompts:
+  - file://../prompts/chat.json
+
+tests:
+  - vars:
+      name: ChatParseTest
+    assert:
+      - type: contains
+        value: ChatParseTest

test/smoke/fixtures/configs/json-prompt.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing JSON chat format prompts
+description: 'Smoke test - JSON chat format prompt'
+
+providers:
+  - echo
+
+prompts:
+  - file://../prompts/chat.json
+
+tests:
+  - vars:
+      name: ChatTest
+    assert:
+      - type: contains
+        value: ChatTest

test/smoke/fixtures/configs/json-schema-assertion.yaml

@@ -0,0 +1,25 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing contains-json with JSON Schema validation
+description: 'Smoke test - JSON schema validation via contains-json'
+
+providers:
+  - echo
+
+prompts:
+  - '{"name": "John", "age": 30}'
+
+tests:
+  - assert:
+      - type: is-json
+      # contains-json with a value validates against JSON Schema
+      - type: contains-json
+        value:
+          type: object
+          required:
+            - name
+            - age
+          properties:
+            name:
+              type: string
+            age:
+              type: number

test/smoke/fixtures/configs/json-tests.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JSON test loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'Greet {{name}}'
+
+tests: file://tests.json

test/smoke/fixtures/configs/jsonl-tests.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JSONL test loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'Greet {{name}}'
+
+tests: file://tests.jsonl

test/smoke/fixtures/configs/latency-assertion.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing latency assertion
+description: 'Smoke test - latency assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test prompt for latency check'
+
+tests:
+  - assert:
+      # Echo provider is fast, should complete under 5000ms
+      - type: latency
+        threshold: 5000

test/smoke/fixtures/configs/levenshtein-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing levenshtein (edit distance) assertion
+description: 'Smoke test - levenshtein assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello World'
+
+tests:
+  - assert:
+      # Allow up to 3 character edits
+      - type: levenshtein
+        value: 'Hello Wurld'
+        threshold: 3

test/smoke/fixtures/configs/llm-rubric-stateful-grader-assert-set-order.yaml

@@ -0,0 +1,73 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Repro for stateful Ollama-style judge model switches inside assert-set
+
+prompts:
+  - 'Answer this prompt: {{topic}}'
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      topic: alpha
+    assert:
+      - type: assert-set
+        assert:
+          - type: llm-rubric
+            value: Judge alpha with model one
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-one
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+          - type: llm-rubric
+            value: Judge alpha with model two
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-two
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: beta
+    assert:
+      - type: assert-set
+        assert:
+          - type: llm-rubric
+            value: Judge beta with model one
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-one
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+          - type: llm-rubric
+            value: Judge beta with model two
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-two
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: gamma
+    assert:
+      - type: assert-set
+        assert:
+          - type: llm-rubric
+            value: Judge gamma with model one
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-one
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+          - type: llm-rubric
+            value: Judge gamma with model two
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-two
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'

test/smoke/fixtures/configs/llm-rubric-stateful-grader-order.yaml

@@ -0,0 +1,67 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Repro for stateful Ollama-style judge model switches
+
+prompts:
+  - 'Answer this prompt: {{topic}}'
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      topic: alpha
+    assert:
+      - type: llm-rubric
+        value: Judge alpha with model one
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-one
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+      - type: llm-rubric
+        value: Judge alpha with model two
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-two
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: beta
+    assert:
+      - type: llm-rubric
+        value: Judge beta with model one
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-one
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+      - type: llm-rubric
+        value: Judge beta with model two
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-two
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: gamma
+    assert:
+      - type: llm-rubric
+        value: Judge gamma with model one
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-one
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+      - type: llm-rubric
+        value: Judge gamma with model two
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-two
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'

test/smoke/fixtures/configs/max-concurrency-config.yaml

@@ -0,0 +1,32 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.3.2: maxConcurrency in config.yaml is respected
+# Bug #6526: maxConcurrency from config file was ignored in 0.120.0
+description: 'Regression test - maxConcurrency from config file'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{n}}'
+
+# This setting was ignored in 0.120.0 - only CLI --max-concurrency worked
+defaultTest:
+  options:
+    maxConcurrency: 1
+
+tests:
+  - vars:
+      n: '1'
+    assert:
+      - type: contains
+        value: '1'
+  - vars:
+      n: '2'
+    assert:
+      - type: contains
+        value: '2'
+  - vars:
+      n: '3'
+    assert:
+      - type: contains
+        value: '3'

test/smoke/fixtures/configs/multi-assertion.yaml

@@ -0,0 +1,15 @@
+description: 'Test multiple assertion types'
+providers:
+  - echo
+prompts:
+  - 'Hello World 123'
+tests:
+  - assert:
+      - type: contains
+        value: Hello
+      - type: contains
+        value: World
+      - type: regex
+        value: '\d+'
+      - type: javascript
+        value: output.length > 5

test/smoke/fixtures/configs/multi-prompt.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing multiple prompts
+description: 'Smoke test - multiple prompts comparison'
+
+providers:
+  - echo
+
+prompts:
+  - 'Greeting: Hello {{name}}!'
+  - 'Greeting: Hi {{name}}!'
+  - 'Greeting: Hey {{name}}!'
+
+tests:
+  - vars:
+      name: Alice
+    assert:
+      - type: contains
+        value: Alice

test/smoke/fixtures/configs/multi-provider.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with multiple providers for provider filtering
+description: 'Smoke test - multiple providers for filtering'
+
+providers:
+  - id: echo
+    label: 'Echo Provider'
+  - id: echo
+    label: 'Custom Echo'
+  - id: echo
+    label: 'Test Provider'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: World

test/smoke/fixtures/configs/multi-test.yaml

@@ -0,0 +1,68 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with multiple tests for filter testing
+description: 'Smoke test - multiple tests for filtering'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, you are a {{role}}'
+
+tests:
+  - description: 'user authentication test'
+    vars:
+      name: Alice
+      role: admin
+    metadata:
+      category: auth
+      priority: high
+    assert:
+      - type: contains
+        value: Alice
+
+  - description: 'user profile test'
+    vars:
+      name: Bob
+      role: user
+    metadata:
+      category: profile
+      priority: medium
+    assert:
+      - type: contains
+        value: Bob
+
+  - description: 'admin dashboard test'
+    vars:
+      name: Charlie
+      role: admin
+    metadata:
+      category: admin
+      priority: high
+    assert:
+      - type: contains
+        value: Charlie
+
+  - description: 'guest access test'
+    vars:
+      name: Diana
+      role: guest
+    metadata:
+      category: auth
+      priority: low
+    assert:
+      - type: contains
+        value: Diana
+
+  - description: 'settings update test'
+    vars:
+      name: Eve
+      role: user
+    metadata:
+      category: settings
+      priority: medium
+      tags:
+        - security
+        - config
+    assert:
+      - type: contains
+        value: Eve

test/smoke/fixtures/configs/normalized-tools.yaml

@@ -0,0 +1,62 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - OpenAI tool format with cross-provider transformation'
+
+providers:
+  # Test OpenAI tools with auto mode
+  - id: 'exec:node ../scripts/echo-config.js'
+    label: 'Auto tool choice'
+    config:
+      tools: &tools
+        - type: function
+          function:
+            name: get_weather
+            description: Get current weather for a location
+            parameters:
+              type: object
+              properties:
+                location:
+                  type: string
+                  description: City name
+                unit:
+                  type: string
+                  enum: [celsius, fahrenheit]
+              required:
+                - location
+        - type: function
+          function:
+            name: search_web
+            description: Search the web for information
+            parameters:
+              type: object
+              properties:
+                query:
+                  type: string
+              required:
+                - query
+      tool_choice: auto
+
+  # Test tool reuse with YAML alias and required mode
+  - id: 'exec:node ../scripts/echo-config.js'
+    label: 'Required tool choice'
+    config:
+      tools: *tools
+      tool_choice: required
+
+  # Test specific tool choice
+  - id: 'exec:node ../scripts/echo-config.js'
+    label: 'Specific tool choice'
+    config:
+      tools: *tools
+      tool_choice:
+        type: function
+        function:
+          name: get_weather
+
+prompts:
+  - 'What is the weather in {{location}}?'
+
+tests:
+  - vars:
+      location: San Francisco
+    assert:
+      - type: is-json

test/smoke/fixtures/configs/not-contains-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - not-contains assertion (5.1.2)'
+
+providers:
+  - echo
+
+prompts:
+  - 'The quick brown fox jumps over the lazy dog'
+
+tests:
+  - vars: {}
+    assert:
+      - type: not-contains
+        value: 'elephant'
+      - type: not-contains
+        value: 'giraffe'

test/smoke/fixtures/configs/not-script-assertions.yaml

@@ -0,0 +1,49 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - negated script assertions'
+
+providers:
+  - echo
+
+prompts:
+  - '{{value}}'
+
+tests:
+  - description: not-javascript inverts a boolean return
+    vars:
+      value: safe-output
+    assert:
+      - type: not-javascript
+        value: output.includes('forbidden')
+
+  - description: not-python preserves score when a numeric return is below threshold
+    vars:
+      value: safe-output
+    assert:
+      - type: not-python
+        value: '0.25'
+        threshold: 0.5
+
+  - description: not-python inverts a JSON-stringified low-score GradingResult
+    vars:
+      value: safe-output
+    assert:
+      - type: not-python
+        value: |
+          return '{"pass": true, "score": 0.25, "reason": "Python raw low score"}'
+        threshold: 0.5
+
+  - description: not-ruby inverts an object return
+    vars:
+      value: safe-output
+    assert:
+      - type: not-ruby
+        value: "{ pass_: false, score: 0.4, reason: 'Ruby raw false' }"
+
+  - description: not-ruby inverts a JSON-stringified low-score GradingResult
+    vars:
+      value: safe-output
+    assert:
+      - type: not-ruby
+        value: |
+          return '{"pass": true, "score": 0.4, "reason": "Ruby raw low score"}'
+        threshold: 0.5

test/smoke/fixtures/configs/nunjucks-conditionals.yaml

@@ -0,0 +1,16 @@
+description: 'Test Nunjucks conditionals'
+providers:
+  - echo
+prompts:
+  - '{% if premium %}Premium user{% else %}Free user{% endif %}'
+tests:
+  - vars:
+      premium: true
+    assert:
+      - type: contains
+        value: Premium
+  - vars:
+      premium: false
+    assert:
+      - type: contains
+        value: Free

test/smoke/fixtures/configs/nunjucks-filters.yaml

@@ -0,0 +1,11 @@
+description: 'Test Nunjucks filters'
+providers:
+  - echo
+prompts:
+  - 'Hello {{ name | upper }}'
+tests:
+  - vars:
+      name: world
+    assert:
+      - type: contains
+        value: WORLD

test/smoke/fixtures/configs/openai-custom-api-key-envar.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - OpenAI assistant custom apiKeyEnvar missing key error'
+
+providers:
+  - id: openai:assistant:test-assistant-id
+    label: 'OpenAI assistant with custom key'
+    config:
+      # Keep evaluation running so the missing-key message is captured in the output JSON.
+      apiKeyRequired: false
+      apiKeyEnvar: CUSTOM_ASSISTANT_KEY
+
+prompts:
+  - 'Hello from smoke test'
+
+tests:
+  - vars: {}

test/smoke/fixtures/configs/prompt-modification.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing prompt prefix/suffix
+description: 'Smoke test - prompt prefix and suffix'
+
+providers:
+  - echo
+
+prompts:
+  - 'MIDDLE'
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: MIDDLE

test/smoke/fixtures/configs/prompt-template.txt

@@ -0,0 +1 @@
+Tell me about {{topic}} in one sentence.

test/smoke/fixtures/configs/provider-cjs.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - CJS provider class'
+
+providers:
+  - file://../providers/echo-cjs.cjs
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: CJS
+    assert:
+      - type: contains
+        value: 'CJS Echo:'
+      - type: contains
+        value: CJS

test/smoke/fixtures/configs/provider-esm.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - ESM provider class'
+
+providers:
+  - file://../providers/echo-esm.mjs
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: ESM
+    assert:
+      - type: contains
+        value: 'ESM Echo:'
+      - type: contains
+        value: ESM

test/smoke/fixtures/configs/provider-label.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing provider label in output
+description: 'Smoke test - provider label'
+
+providers:
+  - id: echo
+    label: 'My Custom Echo Provider'
+
+prompts:
+  - 'Test prompt'
+
+tests:
+  - assert:
+      - type: contains
+        value: 'Test'

test/smoke/fixtures/configs/provider-python-named.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python provider with named function (3.4.2)'
+
+providers:
+  - file://../providers/echo_provider_named.py:custom_echo
+
+prompts:
+  - 'Test prompt: {{input}}'
+
+tests:
+  - vars:
+      input: hello python
+    assert:
+      - type: contains
+        value: 'Python Custom Echo:'

test/smoke/fixtures/configs/provider-ts-transitive.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - TypeScript provider with transitive helper import'
+
+providers:
+  - file://../providers/echo-ts-transitive.ts
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: helper
+    assert:
+      - type: contains
+        value: 'TypeScript Transitive Echo:'

test/smoke/fixtures/configs/provider-ts.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - TypeScript provider class'
+
+providers:
+  - file://../providers/echo-ts.ts
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: TypeScript
+    assert:
+      - type: contains
+        value: 'TypeScript Echo:'
+      - type: contains
+        value: TypeScript

test/smoke/fixtures/configs/provider-with-config.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - provider with config options (7.1.1)'
+
+providers:
+  - id: echo
+    label: 'Custom Echo Provider'
+    config:
+      customOption: 'test-value'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: ConfigTest
+    assert:
+      - type: contains
+        value: 'ConfigTest'

test/smoke/fixtures/configs/providers-with-config.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with multiple providers that have labels and config options.
+# Uses ProviderOptionsMap format with overridden IDs so that the bare
+# string 'echo' does NOT match any config provider by id/label/suffix,
+# enabling a true fallback-to-bare-token smoke test.
+description: 'Smoke test - providers flag preserves config'
+
+providers:
+  - echo:
+      id: 'alpha-echo'
+      label: 'provider-alpha'
+      config:
+        temperature: 0.1
+        custom_option: 'alpha-value'
+  - echo:
+      id: 'beta-echo'
+      label: 'provider-beta'
+      config:
+        temperature: 0.9
+        custom_option: 'beta-value'
+  - echo:
+      id: 'gamma-echo'
+      label: 'provider-gamma'
+      config:
+        temperature: 0.5
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: World

test/smoke/fixtures/configs/python-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python file assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, welcome to the system'
+
+tests:
+  - vars:
+      name: Alice
+      expected_word: Alice
+    assert:
+      - type: python
+        value: file://../assertions/check_keywords.py

test/smoke/fixtures/configs/python-prompt-with-extension.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python prompt function with Python extension hook (issue #9653)'
+
+extensions:
+  - file://../scripts/extension_hook.py:extension_hook
+
+prompts:
+  - id: file://../prompts/create_prompt.py:create_prompt
+    label: hooked-prompt
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      topic: Linear Algebra

test/smoke/fixtures/configs/python-provider.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python provider'
+
+providers:
+  - file://../providers/echo_provider.py
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: Python
+    assert:
+      - type: contains
+        value: 'Python Echo:'
+      - type: contains
+        value: Python

test/smoke/fixtures/configs/range-command-line-options.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://promptfoo.dev/config-schema.json",
+  "commandLineOptions": {
+    "filterRange": "1:3"
+  },
+  "prompts": ["Hello {{name}}"],
+  "providers": ["echo"],
+  "tests": [
+    {
+      "vars": {
+        "name": "Alice"
+      }
+    },
+    {
+      "vars": {
+        "name": "Bob"
+      }
+    },
+    {
+      "vars": {
+        "name": "Charlie"
+      }
+    },
+    {
+      "vars": {
+        "name": "David"
+      }
+    }
+  ]
+}

test/smoke/fixtures/configs/redteam-class-provider-7353.yaml

@@ -0,0 +1,25 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test config for #7353 - class-based provider with prototype id() method in redteam flow
+# This verifies that class-based providers work correctly in the redteam generate flow.
+#
+# The bug was: wrapProviderWithRateLimiting used spread operator which doesn't copy
+# prototype methods like id(), causing "TypeError: redteamProvider.id is not a function"
+# in strategies that call TokenUsageTracker.trackUsage(provider.id(), ...).
+#
+# Uses contracts plugin (local generation) and base64 strategy (local transform)
+# so the test completes without any remote API calls.
+description: 'Test class-based redteam provider with prototype id()'
+
+targets:
+  - id: echo
+    label: echo-target
+
+redteam:
+  # Use our class-based provider as the redteam/attacker provider
+  provider: file://../providers/redteam-class-provider-7353.js
+  purpose: Test application for security vulnerabilities
+  numTests: 1
+  plugins:
+    - contracts
+  strategies:
+    - id: base64

test/smoke/fixtures/configs/resume-many-tests.yaml

@@ -0,0 +1,52 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with many tests for resume E2E testing
+# Uses delay + echo provider so we can reliably interrupt and resume
+description: 'E2E test - resume with many test cases'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}} - item {{num}}'
+
+tests:
+  - vars: { name: Alice, num: '1' }
+    assert:
+      - type: contains
+        value: Alice
+  - vars: { name: Bob, num: '2' }
+    assert:
+      - type: contains
+        value: Bob
+  - vars: { name: Charlie, num: '3' }
+    assert:
+      - type: contains
+        value: Charlie
+  - vars: { name: Diana, num: '4' }
+    assert:
+      - type: contains
+        value: Diana
+  - vars: { name: Eve, num: '5' }
+    assert:
+      - type: contains
+        value: Eve
+  - vars: { name: Frank, num: '6' }
+    assert:
+      - type: contains
+        value: Frank
+  - vars: { name: Grace, num: '7' }
+    assert:
+      - type: contains
+        value: Grace
+  - vars: { name: Heidi, num: '8' }
+    assert:
+      - type: contains
+        value: Heidi
+  - vars: { name: Ivan, num: '9' }
+    assert:
+      - type: contains
+        value: Ivan
+  - vars: { name: Judy, num: '10' }
+    assert:
+      - type: contains
+        value: Judy

test/smoke/fixtures/configs/ruby-provider.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.5.2: Ruby provider works after ESM migration
+# Bug #6506: Ruby provider wrapper failed in 0.120.0
+description: 'Regression test - Ruby provider'
+
+providers:
+  - file://../providers/echo-ruby.rb
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: RubyTest
+    assert:
+      - type: contains
+        value: 'Ruby Echo:'
+      - type: contains
+        value: RubyTest

test/smoke/fixtures/configs/scenarios.yaml

@@ -0,0 +1,40 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - scenarios feature (7.3.1)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, welcome to {{location}}'
+
+scenarios:
+  - description: 'US users scenario'
+    config:
+      # Config must be an array of var combinations
+      - vars:
+          region: 'US'
+    tests:
+      - vars:
+          name: Alice
+          location: New York
+        assert:
+          - type: contains
+            value: 'New York'
+      - vars:
+          name: Bob
+          location: California
+        assert:
+          - type: contains
+            value: 'California'
+
+  - description: 'EU users scenario'
+    config:
+      - vars:
+          region: 'EU'
+    tests:
+      - vars:
+          name: Claude
+          location: Paris
+        assert:
+          - type: contains
+            value: 'Paris'

test/smoke/fixtures/configs/schema-validation-7096.yaml

@@ -0,0 +1,33 @@
+# Fixture for #7096 regression test - JSON Schema validation
+# Tests that options and metadata fields work with combined/custom properties
+prompts:
+  - 'Answer: {{question}}'
+
+providers:
+  - echo
+
+# Test combined options from multiple merged schemas
+defaultTest:
+  options:
+    # From PromptConfigSchema
+    prefix: 'You are helpful.'
+    suffix: 'Be concise.'
+    # From OutputConfigSchema
+    transform: 'output.trim()'
+    # From GradingConfigSchema
+    provider: openai:gpt-4o-mini
+    # Additional properties
+    disableVarExpansion: false
+
+tests:
+  - vars:
+      question: 'What is 2+2?'
+    # Test metadata with custom keys alongside internal properties
+    metadata:
+      customTag: 'math-test'
+      experimentId: 12345
+      pluginConfig:
+        someOption: true
+    assert:
+      - type: contains
+        value: '2+2'