chore: import upstream snapshot with attribution

test/smoke/fixtures/assertions/check-length.js

@@ -0,0 +1,22 @@
+/**
+ * JavaScript assertion file (5.2.2)
+ * Checks that output meets minimum length requirement
+ */
+module.exports = (output, context) => {
+  const minLength = context.test?.assert?.[0]?.config?.minLength || 5;
+  const actualLength = output.length;
+
+  if (actualLength >= minLength) {
+    return {
+      pass: true,
+      score: 1.0,
+      reason: `Output length (${actualLength}) meets minimum (${minLength})`,
+    };
+  }
+
+  return {
+    pass: false,
+    score: actualLength / minLength,
+    reason: `Output length (${actualLength}) below minimum (${minLength})`,
+  };
+};

test/smoke/fixtures/assertions/check_keywords.py

@@ -0,0 +1,22 @@
+"""
+Python assertion file (5.2.5)
+Checks that output contains expected keywords
+"""
+
+
+def get_assert(output, context):
+    """Check if output contains the expected keyword."""
+    expected = context.get("test", {}).get("vars", {}).get("expected_word", "hello")
+
+    if expected.lower() in output.lower():
+        return {
+            "pass": True,
+            "score": 1.0,
+            "reason": f"Output contains '{expected}'",
+        }
+
+    return {
+        "pass": False,
+        "score": 0.0,
+        "reason": f"Output does not contain '{expected}'",
+    }

test/smoke/fixtures/assertions/dynamic-value.js

@@ -0,0 +1,11 @@
+/**
+ * Dynamic assertion value script (#6253)
+ *
+ * This script returns a value to be used in an assertion.
+ * Tests that file:// references in assertion values use script output.
+ */
+
+module.exports = function () {
+  // Return a dynamic value that the assertion will use
+  return 'DynamicValue';
+};

test/smoke/fixtures/assertions/dynamic-value.py

@@ -0,0 +1,10 @@
+"""
+Dynamic assertion value script (#6253)
+
+This script returns a value to be used in an assertion.
+Tests that file:// references in assertion values use script output.
+"""
+
+
+def get_value():
+    return "PythonDynamicValue"

test/smoke/fixtures/configs/assert-dynamic-var.js

@@ -0,0 +1,31 @@
+// Assertion function that verifies dynamic vars are resolved
+// Tests fix for GitHub issue #7334
+module.exports = function (_output, context) {
+  const dynamicVar = context.vars.DYNAMIC_VAR;
+
+  // Check if the variable was resolved (should be an ISO date string)
+  // or still has the file:// prefix (bug)
+  if (typeof dynamicVar === 'string' && dynamicVar.startsWith('file://')) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `BUG: DYNAMIC_VAR was not resolved! Got raw file path: ${dynamicVar}`,
+    };
+  }
+
+  // Validate it's a valid ISO date
+  const date = new Date(dynamicVar);
+  if (isNaN(date.getTime())) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `DYNAMIC_VAR is not a valid date: ${dynamicVar}`,
+    };
+  }
+
+  return {
+    pass: true,
+    score: 1,
+    reason: `DYNAMIC_VAR was correctly resolved to: ${dynamicVar}`,
+  };
+};

test/smoke/fixtures/configs/assertions.yaml

@@ -0,0 +1,50 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - various assertion types'
+
+providers:
+  - echo
+
+prompts:
+  - '{{input}}'
+
+tests:
+  # Test equals assertion
+  - vars:
+      input: 'exact match test'
+    assert:
+      - type: equals
+        value: 'exact match test'
+
+  # Test regex assertion
+  - vars:
+      input: 'The answer is 42'
+    assert:
+      - type: regex
+        value: 'answer.*\d+'
+
+  # Test starts-with assertion
+  - vars:
+      input: 'Hello World'
+    assert:
+      - type: starts-with
+        value: 'Hello'
+
+  # Test is-json assertion
+  - vars:
+      input: '{"key": "value", "number": 123}'
+    assert:
+      - type: is-json
+
+  # Test not-contains assertion
+  - vars:
+      input: 'This is safe content'
+    assert:
+      - type: not-contains
+        value: 'dangerous'
+
+  # Test inline JavaScript assertion
+  - vars:
+      input: 'count to five: 1 2 3 4 5'
+    assert:
+      - type: javascript
+        value: output.includes('5') && output.length > 10

test/smoke/fixtures/configs/basic.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://promptfoo.dev/config-schema.json",
+  "description": "Smoke test - JSON config format",
+  "providers": ["echo"],
+  "prompts": ["Hello {{name}}"],
+  "tests": [
+    {
+      "vars": { "name": "JSON" },
+      "assert": [{ "type": "contains", "value": "Hello" }, { "type": "contains", "value": "JSON" }]
+    }
+  ]
+}

test/smoke/fixtures/configs/basic.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - basic config validation'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: Hello
+      - type: contains
+        value: World

test/smoke/fixtures/configs/circular-ref-7266.yaml

@@ -0,0 +1,23 @@
+# Test config for GitHub issue #7266
+# Provider returns data with circular references (simulating leaked Timeout objects)
+# This should fail on main but pass with the fix
+#
+# NOTE: This test intentionally uses a custom provider instead of 'echo' because:
+# - The echo provider cannot generate circular references in its output
+# - This test must verify that circular reference objects (like Node.js Timeout)
+#   are properly sanitized before database storage
+# - The custom provider creates a controlled circular reference structure
+#   (similar to _idlePrev/_idleNext in Node.js timers) to reproduce the bug
+description: 'Regression test for #7266 - circular reference in provider response'
+
+providers:
+  - file://../providers/circular-ref-provider.js
+
+prompts:
+  - 'Test prompt'
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: 'Processed'

test/smoke/fixtures/configs/cjs-module-exports.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.1.1: CJS provider with module.exports still works after ESM migration
+# Bug #6501: .js files with CJS syntax failed to load in 0.120.0
+description: 'Regression test - CJS module.exports provider'
+
+providers:
+  - file://../providers/cjs-module-exports.js
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: CjsTest
+    assert:
+      - type: contains
+        value: 'CJS Echo:'
+      - type: contains
+        value: CjsTest

test/smoke/fixtures/configs/cjs-with-require.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.1.2: CJS provider that uses require() internally
+# Bug #6468: require() resolution was broken in 0.120.0
+description: 'Regression test - CJS provider with require()'
+
+providers:
+  - file://../providers/cjs-with-require.js
+
+prompts:
+  - 'Test {{value}}'
+
+tests:
+  - vars:
+      value: RequireTest
+    assert:
+      - type: contains
+        value: 'Require Test:'
+      - type: contains
+        value: 'platform='
+      - type: contains
+        value: RequireTest

test/smoke/fixtures/configs/class-provider-7353.yaml

@@ -0,0 +1,16 @@
+# Test config for #7353 - class-based provider with prototype id() method
+# This verifies that class-based providers work correctly through the CLI.
+description: 'Test class-based provider with prototype id()'
+
+providers:
+  - file://../providers/class-provider-prototype-id.js
+
+prompts:
+  - 'Hello World'
+
+tests:
+  - assert:
+      - type: contains
+        value: ClassProvider
+      - type: contains
+        value: Hello World

test/smoke/fixtures/configs/config-base.yaml

@@ -0,0 +1,9 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Base config for testing multiple config merging
+description: 'Base config'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'

test/smoke/fixtures/configs/config-cjs.cjs

@@ -0,0 +1,17 @@
+/**
+ * CJS config using module.exports object (2.3.1)
+ */
+module.exports = {
+  description: 'Smoke test - CJS config format',
+  providers: ['echo'],
+  prompts: ['Hello from CJS config: {{name}}'],
+  tests: [
+    {
+      vars: { name: 'CommonJS' },
+      assert: [
+        { type: 'contains', value: 'Hello from CJS' },
+        { type: 'contains', value: 'CommonJS' },
+      ],
+    },
+  ],
+};

test/smoke/fixtures/configs/config-esm.mjs

@@ -0,0 +1,17 @@
+/**
+ * ESM config using export default (2.3.5)
+ */
+export default {
+  description: 'Smoke test - ESM config format',
+  providers: ['echo'],
+  prompts: ['Hello from ESM config: {{name}}'],
+  tests: [
+    {
+      vars: { name: 'ESModule' },
+      assert: [
+        { type: 'contains', value: 'Hello from ESM' },
+        { type: 'contains', value: 'ESModule' },
+      ],
+    },
+  ],
+};

test/smoke/fixtures/configs/config-tests.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Complete config for testing config-based features
+description: 'Smoke test - config extension'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: ConfigMergeTest
+    assert:
+      - type: contains
+        value: ConfigMergeTest

test/smoke/fixtures/configs/config-ts.ts

@@ -0,0 +1,34 @@
+/**
+ * TypeScript config using export default (2.4.1)
+ *
+ * Uses inline type to avoid depcheck issues with 'promptfoo' import.
+ * In real usage, you would import from 'promptfoo':
+ * import type { UnifiedConfig } from 'promptfoo';
+ */
+
+interface UnifiedConfig {
+  description?: string;
+  providers: string[];
+  prompts: string[];
+  tests: Array<{
+    vars: Record<string, string>;
+    assert: Array<{ type: string; value: string }>;
+  }>;
+}
+
+const config: UnifiedConfig = {
+  description: 'Smoke test - TypeScript config format',
+  providers: ['echo'],
+  prompts: ['Hello from TypeScript config: {{name}}'],
+  tests: [
+    {
+      vars: { name: 'TypeScript' },
+      assert: [
+        { type: 'contains', value: 'Hello from TypeScript' },
+        { type: 'contains', value: 'TypeScript' },
+      ],
+    },
+  ],
+};
+
+export default config;

test/smoke/fixtures/configs/contains-all-assertion.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing contains-all assertion
+description: 'Smoke test - contains-all assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello World, this is a test message'
+
+tests:
+  - assert:
+      # All values must be present
+      - type: contains-all
+        value:
+          - Hello
+          - World
+          - test

test/smoke/fixtures/configs/contains-any-assertion.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing contains-any assertion
+description: 'Smoke test - contains-any assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'The quick brown fox'
+
+tests:
+  - assert:
+      # At least one value must be present
+      - type: contains-any
+        value:
+          - elephant
+          - fox
+          - giraffe

test/smoke/fixtures/configs/contains-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - contains assertion (5.1.1)'
+
+providers:
+  - echo
+
+prompts:
+  - 'The quick brown fox jumps over the lazy dog'
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: 'quick brown'
+      - type: contains
+        value: 'lazy dog'

test/smoke/fixtures/configs/contains-json-assertion.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - contains-json assertion (5.1.7)'
+
+providers:
+  - echo
+
+prompts:
+  # The echo provider echoes the prompt, so include JSON in the prompt
+  - 'Here is the data: {"status": "success", "code": 200, "message": "Hello"}'
+
+tests:
+  - vars: {}
+    assert:
+      # contains-json without value just checks that valid JSON is present in output
+      - type: contains-json
+      # With a value, it validates against a JSON schema
+      - type: contains-json
+        value:
+          type: object
+          required:
+            - status
+            - code
+          properties:
+            status:
+              type: string
+            code:
+              type: number

test/smoke/fixtures/configs/contains-quoted-commas.yaml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing quoted comma parsing across all contains assertion types
+description: 'Smoke test - quoted comma parsing in contains assertions'
+
+providers:
+  - echo
+
+prompts:
+  - 'hello, world and foo bar'
+
+tests:
+  # contains-any with quoted comma value (string format)
+  - assert:
+      - type: contains-any
+        value: '"hello, world",universe'
+
+  # contains-all with quoted comma values (string format)
+  - assert:
+      - type: contains-all
+        value: '"hello, world","foo"'
+
+  # icontains-any with quoted comma value
+  - assert:
+      - type: icontains-any
+        value: '"HELLO, WORLD",universe'
+
+  # icontains-all with quoted comma values
+  - assert:
+      - type: icontains-all
+        value: '"HELLO, WORLD","FOO"'

test/smoke/fixtures/configs/conversation-relevance-ssti.yaml

@@ -0,0 +1,23 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - conversation relevance SSTI'
+
+prompts:
+  - 'unused prompt'
+
+providers:
+  - echo
+
+defaultTest:
+  options:
+    provider: 'exec:node ../scripts/capture-conversation-relevance-grader.js'
+
+tests:
+  - vars:
+      country: France
+      capital: Paris
+      _conversation:
+        - input: 'What is the capital of {{country}}?'
+          output: 'The answer is {{capital}} and {{ env.OPENAI_API_KEY }} and {% for x in range(3) %}{{x}}{% endfor %}'
+    assert:
+      - type: conversation-relevance
+        threshold: 0

test/smoke/fixtures/configs/cost-assertion.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing cost assertion
+description: 'Smoke test - cost assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test prompt for cost check'
+
+tests:
+  - assert:
+      # Echo provider has 0 cost, should pass with any threshold
+      - type: cost
+        threshold: 1.0

test/smoke/fixtures/configs/csv-tests.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - CSV test loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, you are {{age}} years old'
+
+tests: file://tests.csv

test/smoke/fixtures/configs/default-test.yaml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - defaultTest feature (7.2.1)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+# Default assertions applied to all tests
+defaultTest:
+  assert:
+    - type: contains
+      value: 'Hello'
+
+tests:
+  - vars:
+      name: Alice
+  - vars:
+      name: Bob
+  - vars:
+      name: Charlie

test/smoke/fixtures/configs/defaulttest-llm-rubric-vars-default.yaml

@@ -0,0 +1,7 @@
+options:
+  provider:
+    id: file://../providers/defaulttest-llm-rubric-grader.cjs
+
+assert:
+  - type: llm-rubric
+    value: 'Does the output correctly reference the input: {{myVar}}?'

test/smoke/fixtures/configs/defaulttest-llm-rubric-vars-tests.yaml

@@ -0,0 +1,4 @@
+- vars:
+    myVar: 'hello world'
+- vars:
+    myVar: 'goodbye moon'

test/smoke/fixtures/configs/defaulttest-llm-rubric-vars.yaml

@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Regression test - defaultTest llm-rubric vars'
+
+prompts:
+  - 'static model output'
+
+providers:
+  - echo
+
+defaultTest: file://defaulttest-llm-rubric-vars-default.yaml
+
+tests:
+  - file://defaulttest-llm-rubric-vars-tests.yaml

test/smoke/fixtures/configs/delay-test.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Simple config for testing --delay flag
+description: 'Smoke test - delay between tests'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{num}}'
+
+tests:
+  - vars:
+      num: '1'
+  - vars:
+      num: '2'
+  - vars:
+      num: '3'

test/smoke/fixtures/configs/dynamic-var-assertion-7334.yaml

@@ -0,0 +1,23 @@
+# Regression test for GitHub Issue #7334
+# Dynamic variables should be resolved when passed to assertion functions
+# https://github.com/promptfoo/promptfoo/issues/7334
+
+providers:
+  - echo
+
+prompts:
+  - 'The dynamic variable is: {{DYNAMIC_VAR}}'
+
+defaultTest:
+  vars:
+    # This dynamic variable is loaded from a JavaScript file
+    DYNAMIC_VAR: file://dynamic-var-generator.js
+
+tests:
+  - description: 'Dynamic vars should be resolved in assertion context.vars'
+    assert:
+      # This assertion function accesses context.vars.DYNAMIC_VAR
+      # Before fix: received "file://dynamic-var-generator.js"
+      # After fix: receives the resolved ISO date string
+      - type: javascript
+        value: file://assert-dynamic-var.js

test/smoke/fixtures/configs/dynamic-var-generator.js

@@ -0,0 +1,7 @@
+// Dynamic variable that returns current ISO UTC timestamp
+// Used to test that file:// vars are resolved before being passed to assertions
+module.exports = function (_varName, _prompt, _vars, _provider) {
+  return {
+    output: new Date().toISOString(),
+  };
+};

test/smoke/fixtures/configs/empty-vars.yaml

@@ -0,0 +1,10 @@
+description: 'Test empty vars handling'
+providers:
+  - echo
+prompts:
+  - 'Static prompt'
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: Static

test/smoke/fixtures/configs/ends-with-assertion.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing regex assertion with end-of-string pattern
+description: 'Smoke test - regex ends-with pattern'
+
+providers:
+  - echo
+
+prompts:
+  - 'The answer is 42.'
+
+tests:
+  - assert:
+      # Regex pattern to match string ending with "42."
+      - type: regex
+        value: '42\.$'
+      # Regex pattern to match string ending with "."
+      - type: regex
+        value: '\.$'

test/smoke/fixtures/configs/env-var-test.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing --env-file flag
+description: 'Smoke test - environment variable loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'API Key: {{api_key}}, Secret: {{secret}}'
+
+tests:
+  - vars:
+      api_key: '{{env.TEST_API_KEY}}'
+      secret: '{{env.TEST_SECRET}}'
+    assert:
+      - type: contains
+        value: 'API Key:'

test/smoke/fixtures/configs/exec-provider-stdin.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - exec provider closes stdin (regression for #8686)'
+
+providers:
+  - 'exec:node ../providers/exec-provider-reads-stdin.js'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: stdin closed

test/smoke/fixtures/configs/exec-provider.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - exec provider'
+
+providers:
+  - 'exec:echo "Echo from exec: {{prompt}}"'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: Echo from exec

test/smoke/fixtures/configs/failing-assertion.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - config with failing assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      # This assertion will fail because echo returns "Hello World"
+      # but we're asserting it contains "IMPOSSIBLE_STRING_NOT_IN_OUTPUT"
+      - type: contains
+        value: IMPOSSIBLE_STRING_NOT_IN_OUTPUT_12345

test/smoke/fixtures/configs/failing-tests.yaml

@@ -0,0 +1,38 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with some intentionally failing tests for --filter-failing testing
+description: 'Smoke test - config with failing tests'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - description: 'passing test 1'
+    vars:
+      name: Alice
+    assert:
+      - type: contains
+        value: Alice
+
+  - description: 'failing test 1'
+    vars:
+      name: Bob
+    assert:
+      - type: contains
+        value: 'WILL_NOT_MATCH_12345'
+
+  - description: 'passing test 2'
+    vars:
+      name: Charlie
+    assert:
+      - type: contains
+        value: Charlie
+
+  - description: 'failing test 2'
+    vars:
+      name: Diana
+    assert:
+      - type: contains
+        value: 'ALSO_NOT_MATCHING_67890'

test/smoke/fixtures/configs/file-prompt.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - file:// prompt loading'
+
+providers:
+  - echo
+
+prompts:
+  - file://prompt-template.txt
+
+tests:
+  - vars:
+      topic: bananas
+    assert:
+      - type: contains
+        value: bananas

test/smoke/fixtures/configs/file-provider-env-7079.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - file provider env placeholders'
+
+providers:
+  - file://../providers/file-provider-env-7079.yaml
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: World

test/smoke/fixtures/configs/file-ref-assertion-value.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test for #6253: file:// references in assertion values use script output
+description: 'Regression test - file:// references in assertion values'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test DynamicValue here'
+
+tests:
+  - assert:
+      # The file:// reference should execute the script and use its return value
+      - type: contains
+        value: file://../assertions/dynamic-value.js

test/smoke/fixtures/configs/file-ref-vars.yaml

@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test for #6393: file:// references in vars context for runtime loading
+description: 'Regression test - file:// references in vars'
+
+providers:
+  - echo
+
+prompts:
+  - 'Value is {{dynamicVar}}'
+
+# Load tests from a file that includes vars
+tests: file://../data/tests-with-vars.yaml

test/smoke/fixtures/configs/function-provider-defaulttest.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test for #6174: function providers in defaultTest.options.provider
+description: 'Regression test - function providers in defaultTest'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{name}}'
+
+# Function provider in defaultTest.options.provider should work
+defaultTest:
+  assert:
+    - type: contains
+      value: Test
+
+tests:
+  - vars:
+      name: FunctionProvider

test/smoke/fixtures/configs/function-providers-9383.ts

@@ -0,0 +1,35 @@
+/**
+ * Regression fixture for #9383: multiple CallApiFunction providers must each
+ * survive combineConfigs (the JSON.stringify-based dedupe used to collapse them
+ * into a single entry).
+ *
+ * Mirrors the repro from the issue: three labeled function providers that each
+ * embed their label in the response so the eval output is distinguishable.
+ */
+
+interface CallApiFunction {
+  (prompt: string): Promise<{ output: string }>;
+  label?: string;
+}
+
+interface UnifiedConfig {
+  description?: string;
+  providers: CallApiFunction[];
+  prompts: string[];
+  tests: Array<{ vars: Record<string, string> }>;
+}
+
+const makeProvider = (label: string): CallApiFunction => {
+  const fn: CallApiFunction = async (prompt) => ({ output: `${label}: ${prompt}` });
+  fn.label = label;
+  return fn;
+};
+
+const config: UnifiedConfig = {
+  description: 'Smoke test - multiple function providers (#9383)',
+  providers: [makeProvider('a'), makeProvider('b'), makeProvider('c')],
+  prompts: ['{{prompt}}'],
+  tests: [{ vars: { prompt: 'hi' } }],
+};
+
+export default config;

test/smoke/fixtures/configs/glob-prompts.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing multiple prompts from files
+description: 'Smoke test - multiple file prompts'
+
+providers:
+  - echo
+
+prompts:
+  - file://../prompts/greeting.txt
+  - file://../prompts/farewell.txt
+
+tests:
+  - vars:
+      name: GlobTest
+    assert:
+      - type: contains
+        value: GlobTest

test/smoke/fixtures/configs/go-provider.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.5.1: Go provider works after ESM migration
+# Bug #6506: Go provider wrapper failed in 0.120.0
+description: 'Regression test - Go provider'
+
+providers:
+  - file://../providers/echo-go.go
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: GoTest
+    assert:
+      - type: contains
+        value: 'Go Echo:'
+      - type: contains
+        value: GoTest

test/smoke/fixtures/configs/icontains-assertion.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing icontains (case-insensitive contains) assertion
+description: 'Smoke test - icontains assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Response: HELLO World Test'
+
+tests:
+  - assert:
+      # Case-insensitive match - "hello" matches "HELLO"
+      - type: icontains
+        value: hello
+      # Case-insensitive match - "WORLD" matches "World"
+      - type: icontains
+        value: WORLD
+      # Case-insensitive match - mixed case
+      - type: icontains
+        value: wOrLd

test/smoke/fixtures/configs/inline-js-assertion.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing inline JavaScript assertion
+description: 'Smoke test - inline JavaScript assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'The answer is 42'
+
+tests:
+  - assert:
+      # Inline JavaScript expression
+      - type: javascript
+        value: 'output.includes("42") && output.length > 10'

test/smoke/fixtures/configs/inline-js-process.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.1.3: Inline JS with process.mainModule shim
+# Bug #6606: Inline transforms using process.mainModule.require broke in 0.120.0
+#
+# After ESM migration, process.mainModule is undefined. The fix adds a shim
+# that provides process.mainModule.require via createRequire().
+description: 'Regression test - inline JS with process access'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{value}}'
+
+tests:
+  - vars:
+      value: ProcessTest
+    assert:
+      - type: javascript
+        value: |
+          // Test that process object is accessible in inline JS
+          // This broke in 0.120.0 because process.mainModule was undefined
+          // Note: 'output' is already defined in the context, use it directly
+          const hasProcess = typeof process !== 'undefined';
+          const hasEnv = hasProcess && typeof process.env !== 'undefined';
+          // The assertion should pass if process is accessible
+          return output.includes('ProcessTest') && hasProcess && hasEnv;

test/smoke/fixtures/configs/invalid.yaml

@@ -0,0 +1,14 @@
+# This config is intentionally invalid for testing validation errors
+description: 'Invalid config for smoke testing'
+
+# Missing required 'providers' field
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: invalid-assertion-type-that-does-not-exist
+        value: test

test/smoke/fixtures/configs/js-assertion.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JavaScript file assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'This is a test message with some content: {{topic}}'
+
+tests:
+  - vars:
+      topic: bananas
+    assert:
+      - type: javascript
+        value: file://../assertions/check-length.js
+        config:
+          minLength: 10

test/smoke/fixtures/configs/js-test-generator.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JavaScript test generator (4.2.7)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests: file://../data/test-generator.js

test/smoke/fixtures/configs/json-chat-format.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 11.2.1: JSON chat message parsing
+# Bug #6568: Incorrect parsing of JSON vs non-JSON chat messages in 0.120.1
+description: 'Regression test - JSON chat message parsing'
+
+providers:
+  - echo
+
+# This uses a JSON file containing chat messages in array format
+prompts:
+  - file://../prompts/chat.json
+
+tests:
+  - vars:
+      name: ChatParseTest
+    assert:
+      - type: contains
+        value: ChatParseTest

test/smoke/fixtures/configs/json-prompt.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing JSON chat format prompts
+description: 'Smoke test - JSON chat format prompt'
+
+providers:
+  - echo
+
+prompts:
+  - file://../prompts/chat.json
+
+tests:
+  - vars:
+      name: ChatTest
+    assert:
+      - type: contains
+        value: ChatTest

test/smoke/fixtures/configs/json-schema-assertion.yaml

@@ -0,0 +1,25 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing contains-json with JSON Schema validation
+description: 'Smoke test - JSON schema validation via contains-json'
+
+providers:
+  - echo
+
+prompts:
+  - '{"name": "John", "age": 30}'
+
+tests:
+  - assert:
+      - type: is-json
+      # contains-json with a value validates against JSON Schema
+      - type: contains-json
+        value:
+          type: object
+          required:
+            - name
+            - age
+          properties:
+            name:
+              type: string
+            age:
+              type: number

test/smoke/fixtures/configs/json-tests.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JSON test loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'Greet {{name}}'
+
+tests: file://tests.json

test/smoke/fixtures/configs/jsonl-tests.yaml

@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - JSONL test loading'
+
+providers:
+  - echo
+
+prompts:
+  - 'Greet {{name}}'
+
+tests: file://tests.jsonl

test/smoke/fixtures/configs/latency-assertion.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing latency assertion
+description: 'Smoke test - latency assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test prompt for latency check'
+
+tests:
+  - assert:
+      # Echo provider is fast, should complete under 5000ms
+      - type: latency
+        threshold: 5000

test/smoke/fixtures/configs/levenshtein-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing levenshtein (edit distance) assertion
+description: 'Smoke test - levenshtein assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello World'
+
+tests:
+  - assert:
+      # Allow up to 3 character edits
+      - type: levenshtein
+        value: 'Hello Wurld'
+        threshold: 3

test/smoke/fixtures/configs/llm-rubric-stateful-grader-assert-set-order.yaml

@@ -0,0 +1,73 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Repro for stateful Ollama-style judge model switches inside assert-set
+
+prompts:
+  - 'Answer this prompt: {{topic}}'
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      topic: alpha
+    assert:
+      - type: assert-set
+        assert:
+          - type: llm-rubric
+            value: Judge alpha with model one
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-one
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+          - type: llm-rubric
+            value: Judge alpha with model two
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-two
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: beta
+    assert:
+      - type: assert-set
+        assert:
+          - type: llm-rubric
+            value: Judge beta with model one
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-one
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+          - type: llm-rubric
+            value: Judge beta with model two
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-two
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: gamma
+    assert:
+      - type: assert-set
+        assert:
+          - type: llm-rubric
+            value: Judge gamma with model one
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-one
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+          - type: llm-rubric
+            value: Judge gamma with model two
+            provider:
+              id: file://../providers/stateful-ollama-grader.cjs
+              config:
+                modelName: judge-two
+                logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+                reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'

test/smoke/fixtures/configs/llm-rubric-stateful-grader-order.yaml

@@ -0,0 +1,67 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Repro for stateful Ollama-style judge model switches
+
+prompts:
+  - 'Answer this prompt: {{topic}}'
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      topic: alpha
+    assert:
+      - type: llm-rubric
+        value: Judge alpha with model one
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-one
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+      - type: llm-rubric
+        value: Judge alpha with model two
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-two
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: beta
+    assert:
+      - type: llm-rubric
+        value: Judge beta with model one
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-one
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+      - type: llm-rubric
+        value: Judge beta with model two
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-two
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+  - vars:
+      topic: gamma
+    assert:
+      - type: llm-rubric
+        value: Judge gamma with model one
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-one
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'
+      - type: llm-rubric
+        value: Judge gamma with model two
+        provider:
+          id: file://../providers/stateful-ollama-grader.cjs
+          config:
+            modelName: judge-two
+            logPath: '{{ env.PROMPTFOO_STATEFUL_GRADER_LOG_PATH | default("") }}'
+            reloadDelayMs: '{{ env.PROMPTFOO_STATEFUL_GRADER_RELOAD_DELAY_MS | default(0) }}'

test/smoke/fixtures/configs/max-concurrency-config.yaml

@@ -0,0 +1,32 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.3.2: maxConcurrency in config.yaml is respected
+# Bug #6526: maxConcurrency from config file was ignored in 0.120.0
+description: 'Regression test - maxConcurrency from config file'
+
+providers:
+  - echo
+
+prompts:
+  - 'Test {{n}}'
+
+# This setting was ignored in 0.120.0 - only CLI --max-concurrency worked
+defaultTest:
+  options:
+    maxConcurrency: 1
+
+tests:
+  - vars:
+      n: '1'
+    assert:
+      - type: contains
+        value: '1'
+  - vars:
+      n: '2'
+    assert:
+      - type: contains
+        value: '2'
+  - vars:
+      n: '3'
+    assert:
+      - type: contains
+        value: '3'

test/smoke/fixtures/configs/multi-assertion.yaml

@@ -0,0 +1,15 @@
+description: 'Test multiple assertion types'
+providers:
+  - echo
+prompts:
+  - 'Hello World 123'
+tests:
+  - assert:
+      - type: contains
+        value: Hello
+      - type: contains
+        value: World
+      - type: regex
+        value: '\d+'
+      - type: javascript
+        value: output.length > 5

test/smoke/fixtures/configs/multi-prompt.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing multiple prompts
+description: 'Smoke test - multiple prompts comparison'
+
+providers:
+  - echo
+
+prompts:
+  - 'Greeting: Hello {{name}}!'
+  - 'Greeting: Hi {{name}}!'
+  - 'Greeting: Hey {{name}}!'
+
+tests:
+  - vars:
+      name: Alice
+    assert:
+      - type: contains
+        value: Alice

test/smoke/fixtures/configs/multi-provider.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with multiple providers for provider filtering
+description: 'Smoke test - multiple providers for filtering'
+
+providers:
+  - id: echo
+    label: 'Echo Provider'
+  - id: echo
+    label: 'Custom Echo'
+  - id: echo
+    label: 'Test Provider'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: World

test/smoke/fixtures/configs/multi-test.yaml

@@ -0,0 +1,68 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with multiple tests for filter testing
+description: 'Smoke test - multiple tests for filtering'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, you are a {{role}}'
+
+tests:
+  - description: 'user authentication test'
+    vars:
+      name: Alice
+      role: admin
+    metadata:
+      category: auth
+      priority: high
+    assert:
+      - type: contains
+        value: Alice
+
+  - description: 'user profile test'
+    vars:
+      name: Bob
+      role: user
+    metadata:
+      category: profile
+      priority: medium
+    assert:
+      - type: contains
+        value: Bob
+
+  - description: 'admin dashboard test'
+    vars:
+      name: Charlie
+      role: admin
+    metadata:
+      category: admin
+      priority: high
+    assert:
+      - type: contains
+        value: Charlie
+
+  - description: 'guest access test'
+    vars:
+      name: Diana
+      role: guest
+    metadata:
+      category: auth
+      priority: low
+    assert:
+      - type: contains
+        value: Diana
+
+  - description: 'settings update test'
+    vars:
+      name: Eve
+      role: user
+    metadata:
+      category: settings
+      priority: medium
+      tags:
+        - security
+        - config
+    assert:
+      - type: contains
+        value: Eve

test/smoke/fixtures/configs/normalized-tools.yaml

@@ -0,0 +1,62 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - OpenAI tool format with cross-provider transformation'
+
+providers:
+  # Test OpenAI tools with auto mode
+  - id: 'exec:node ../scripts/echo-config.js'
+    label: 'Auto tool choice'
+    config:
+      tools: &tools
+        - type: function
+          function:
+            name: get_weather
+            description: Get current weather for a location
+            parameters:
+              type: object
+              properties:
+                location:
+                  type: string
+                  description: City name
+                unit:
+                  type: string
+                  enum: [celsius, fahrenheit]
+              required:
+                - location
+        - type: function
+          function:
+            name: search_web
+            description: Search the web for information
+            parameters:
+              type: object
+              properties:
+                query:
+                  type: string
+              required:
+                - query
+      tool_choice: auto
+
+  # Test tool reuse with YAML alias and required mode
+  - id: 'exec:node ../scripts/echo-config.js'
+    label: 'Required tool choice'
+    config:
+      tools: *tools
+      tool_choice: required
+
+  # Test specific tool choice
+  - id: 'exec:node ../scripts/echo-config.js'
+    label: 'Specific tool choice'
+    config:
+      tools: *tools
+      tool_choice:
+        type: function
+        function:
+          name: get_weather
+
+prompts:
+  - 'What is the weather in {{location}}?'
+
+tests:
+  - vars:
+      location: San Francisco
+    assert:
+      - type: is-json

test/smoke/fixtures/configs/not-contains-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - not-contains assertion (5.1.2)'
+
+providers:
+  - echo
+
+prompts:
+  - 'The quick brown fox jumps over the lazy dog'
+
+tests:
+  - vars: {}
+    assert:
+      - type: not-contains
+        value: 'elephant'
+      - type: not-contains
+        value: 'giraffe'

test/smoke/fixtures/configs/not-script-assertions.yaml

@@ -0,0 +1,49 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - negated script assertions'
+
+providers:
+  - echo
+
+prompts:
+  - '{{value}}'
+
+tests:
+  - description: not-javascript inverts a boolean return
+    vars:
+      value: safe-output
+    assert:
+      - type: not-javascript
+        value: output.includes('forbidden')
+
+  - description: not-python preserves score when a numeric return is below threshold
+    vars:
+      value: safe-output
+    assert:
+      - type: not-python
+        value: '0.25'
+        threshold: 0.5
+
+  - description: not-python inverts a JSON-stringified low-score GradingResult
+    vars:
+      value: safe-output
+    assert:
+      - type: not-python
+        value: |
+          return '{"pass": true, "score": 0.25, "reason": "Python raw low score"}'
+        threshold: 0.5
+
+  - description: not-ruby inverts an object return
+    vars:
+      value: safe-output
+    assert:
+      - type: not-ruby
+        value: "{ pass_: false, score: 0.4, reason: 'Ruby raw false' }"
+
+  - description: not-ruby inverts a JSON-stringified low-score GradingResult
+    vars:
+      value: safe-output
+    assert:
+      - type: not-ruby
+        value: |
+          return '{"pass": true, "score": 0.4, "reason": "Ruby raw low score"}'
+        threshold: 0.5

test/smoke/fixtures/configs/nunjucks-conditionals.yaml

@@ -0,0 +1,16 @@
+description: 'Test Nunjucks conditionals'
+providers:
+  - echo
+prompts:
+  - '{% if premium %}Premium user{% else %}Free user{% endif %}'
+tests:
+  - vars:
+      premium: true
+    assert:
+      - type: contains
+        value: Premium
+  - vars:
+      premium: false
+    assert:
+      - type: contains
+        value: Free

test/smoke/fixtures/configs/nunjucks-filters.yaml

@@ -0,0 +1,11 @@
+description: 'Test Nunjucks filters'
+providers:
+  - echo
+prompts:
+  - 'Hello {{ name | upper }}'
+tests:
+  - vars:
+      name: world
+    assert:
+      - type: contains
+        value: WORLD

test/smoke/fixtures/configs/openai-custom-api-key-envar.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - OpenAI assistant custom apiKeyEnvar missing key error'
+
+providers:
+  - id: openai:assistant:test-assistant-id
+    label: 'OpenAI assistant with custom key'
+    config:
+      # Keep evaluation running so the missing-key message is captured in the output JSON.
+      apiKeyRequired: false
+      apiKeyEnvar: CUSTOM_ASSISTANT_KEY
+
+prompts:
+  - 'Hello from smoke test'
+
+tests:
+  - vars: {}

test/smoke/fixtures/configs/prompt-modification.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing prompt prefix/suffix
+description: 'Smoke test - prompt prefix and suffix'
+
+providers:
+  - echo
+
+prompts:
+  - 'MIDDLE'
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: MIDDLE

test/smoke/fixtures/configs/prompt-template.txt

@@ -0,0 +1 @@
+Tell me about {{topic}} in one sentence.

test/smoke/fixtures/configs/provider-cjs.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - CJS provider class'
+
+providers:
+  - file://../providers/echo-cjs.cjs
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: CJS
+    assert:
+      - type: contains
+        value: 'CJS Echo:'
+      - type: contains
+        value: CJS

test/smoke/fixtures/configs/provider-esm.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - ESM provider class'
+
+providers:
+  - file://../providers/echo-esm.mjs
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: ESM
+    assert:
+      - type: contains
+        value: 'ESM Echo:'
+      - type: contains
+        value: ESM

test/smoke/fixtures/configs/provider-label.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing provider label in output
+description: 'Smoke test - provider label'
+
+providers:
+  - id: echo
+    label: 'My Custom Echo Provider'
+
+prompts:
+  - 'Test prompt'
+
+tests:
+  - assert:
+      - type: contains
+        value: 'Test'

test/smoke/fixtures/configs/provider-python-named.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python provider with named function (3.4.2)'
+
+providers:
+  - file://../providers/echo_provider_named.py:custom_echo
+
+prompts:
+  - 'Test prompt: {{input}}'
+
+tests:
+  - vars:
+      input: hello python
+    assert:
+      - type: contains
+        value: 'Python Custom Echo:'

test/smoke/fixtures/configs/provider-ts-transitive.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - TypeScript provider with transitive helper import'
+
+providers:
+  - file://../providers/echo-ts-transitive.ts
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: helper
+    assert:
+      - type: contains
+        value: 'TypeScript Transitive Echo:'

test/smoke/fixtures/configs/provider-ts.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - TypeScript provider class'
+
+providers:
+  - file://../providers/echo-ts.ts
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: TypeScript
+    assert:
+      - type: contains
+        value: 'TypeScript Echo:'
+      - type: contains
+        value: TypeScript

test/smoke/fixtures/configs/provider-with-config.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - provider with config options (7.1.1)'
+
+providers:
+  - id: echo
+    label: 'Custom Echo Provider'
+    config:
+      customOption: 'test-value'
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: ConfigTest
+    assert:
+      - type: contains
+        value: 'ConfigTest'

test/smoke/fixtures/configs/providers-with-config.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with multiple providers that have labels and config options.
+# Uses ProviderOptionsMap format with overridden IDs so that the bare
+# string 'echo' does NOT match any config provider by id/label/suffix,
+# enabling a true fallback-to-bare-token smoke test.
+description: 'Smoke test - providers flag preserves config'
+
+providers:
+  - echo:
+      id: 'alpha-echo'
+      label: 'provider-alpha'
+      config:
+        temperature: 0.1
+        custom_option: 'alpha-value'
+  - echo:
+      id: 'beta-echo'
+      label: 'provider-beta'
+      config:
+        temperature: 0.9
+        custom_option: 'beta-value'
+  - echo:
+      id: 'gamma-echo'
+      label: 'provider-gamma'
+      config:
+        temperature: 0.5
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: World

test/smoke/fixtures/configs/python-assertion.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python file assertion'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, welcome to the system'
+
+tests:
+  - vars:
+      name: Alice
+      expected_word: Alice
+    assert:
+      - type: python
+        value: file://../assertions/check_keywords.py

test/smoke/fixtures/configs/python-prompt-with-extension.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python prompt function with Python extension hook (issue #9653)'
+
+extensions:
+  - file://../scripts/extension_hook.py:extension_hook
+
+prompts:
+  - id: file://../prompts/create_prompt.py:create_prompt
+    label: hooked-prompt
+
+providers:
+  - echo
+
+tests:
+  - vars:
+      topic: Linear Algebra

test/smoke/fixtures/configs/python-provider.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - Python provider'
+
+providers:
+  - file://../providers/echo_provider.py
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: Python
+    assert:
+      - type: contains
+        value: 'Python Echo:'
+      - type: contains
+        value: Python

test/smoke/fixtures/configs/range-command-line-options.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://promptfoo.dev/config-schema.json",
+  "commandLineOptions": {
+    "filterRange": "1:3"
+  },
+  "prompts": ["Hello {{name}}"],
+  "providers": ["echo"],
+  "tests": [
+    {
+      "vars": {
+        "name": "Alice"
+      }
+    },
+    {
+      "vars": {
+        "name": "Bob"
+      }
+    },
+    {
+      "vars": {
+        "name": "Charlie"
+      }
+    },
+    {
+      "vars": {
+        "name": "David"
+      }
+    }
+  ]
+}

test/smoke/fixtures/configs/redteam-class-provider-7353.yaml

@@ -0,0 +1,25 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test config for #7353 - class-based provider with prototype id() method in redteam flow
+# This verifies that class-based providers work correctly in the redteam generate flow.
+#
+# The bug was: wrapProviderWithRateLimiting used spread operator which doesn't copy
+# prototype methods like id(), causing "TypeError: redteamProvider.id is not a function"
+# in strategies that call TokenUsageTracker.trackUsage(provider.id(), ...).
+#
+# Uses contracts plugin (local generation) and base64 strategy (local transform)
+# so the test completes without any remote API calls.
+description: 'Test class-based redteam provider with prototype id()'
+
+targets:
+  - id: echo
+    label: echo-target
+
+redteam:
+  # Use our class-based provider as the redteam/attacker provider
+  provider: file://../providers/redteam-class-provider-7353.js
+  purpose: Test application for security vulnerabilities
+  numTests: 1
+  plugins:
+    - contracts
+  strategies:
+    - id: base64

test/smoke/fixtures/configs/resume-many-tests.yaml

@@ -0,0 +1,52 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config with many tests for resume E2E testing
+# Uses delay + echo provider so we can reliably interrupt and resume
+description: 'E2E test - resume with many test cases'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}} - item {{num}}'
+
+tests:
+  - vars: { name: Alice, num: '1' }
+    assert:
+      - type: contains
+        value: Alice
+  - vars: { name: Bob, num: '2' }
+    assert:
+      - type: contains
+        value: Bob
+  - vars: { name: Charlie, num: '3' }
+    assert:
+      - type: contains
+        value: Charlie
+  - vars: { name: Diana, num: '4' }
+    assert:
+      - type: contains
+        value: Diana
+  - vars: { name: Eve, num: '5' }
+    assert:
+      - type: contains
+        value: Eve
+  - vars: { name: Frank, num: '6' }
+    assert:
+      - type: contains
+        value: Frank
+  - vars: { name: Grace, num: '7' }
+    assert:
+      - type: contains
+        value: Grace
+  - vars: { name: Heidi, num: '8' }
+    assert:
+      - type: contains
+        value: Heidi
+  - vars: { name: Ivan, num: '9' }
+    assert:
+      - type: contains
+        value: Ivan
+  - vars: { name: Judy, num: '10' }
+    assert:
+      - type: contains
+        value: Judy

test/smoke/fixtures/configs/ruby-provider.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Test 10.5.2: Ruby provider works after ESM migration
+# Bug #6506: Ruby provider wrapper failed in 0.120.0
+description: 'Regression test - Ruby provider'
+
+providers:
+  - file://../providers/echo-ruby.rb
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: RubyTest
+    assert:
+      - type: contains
+        value: 'Ruby Echo:'
+      - type: contains
+        value: RubyTest

test/smoke/fixtures/configs/scenarios.yaml

@@ -0,0 +1,40 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - scenarios feature (7.3.1)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}, welcome to {{location}}'
+
+scenarios:
+  - description: 'US users scenario'
+    config:
+      # Config must be an array of var combinations
+      - vars:
+          region: 'US'
+    tests:
+      - vars:
+          name: Alice
+          location: New York
+        assert:
+          - type: contains
+            value: 'New York'
+      - vars:
+          name: Bob
+          location: California
+        assert:
+          - type: contains
+            value: 'California'
+
+  - description: 'EU users scenario'
+    config:
+      - vars:
+          region: 'EU'
+    tests:
+      - vars:
+          name: Claude
+          location: Paris
+        assert:
+          - type: contains
+            value: 'Paris'

test/smoke/fixtures/configs/schema-validation-7096.yaml

@@ -0,0 +1,33 @@
+# Fixture for #7096 regression test - JSON Schema validation
+# Tests that options and metadata fields work with combined/custom properties
+prompts:
+  - 'Answer: {{question}}'
+
+providers:
+  - echo
+
+# Test combined options from multiple merged schemas
+defaultTest:
+  options:
+    # From PromptConfigSchema
+    prefix: 'You are helpful.'
+    suffix: 'Be concise.'
+    # From OutputConfigSchema
+    transform: 'output.trim()'
+    # From GradingConfigSchema
+    provider: openai:gpt-4o-mini
+    # Additional properties
+    disableVarExpansion: false
+
+tests:
+  - vars:
+      question: 'What is 2+2?'
+    # Test metadata with custom keys alongside internal properties
+    metadata:
+      customTag: 'math-test'
+      experimentId: 12345
+      pluginConfig:
+        someOption: true
+    assert:
+      - type: contains
+        value: '2+2'

test/smoke/fixtures/configs/simple-repeat.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Simple config for testing --repeat flag with count verification
+description: 'Smoke test - repeat flag'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: RepeatTest
+    assert:
+      - type: contains
+        value: RepeatTest

test/smoke/fixtures/configs/skill-used-provider-esm.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - skill-used assertion with local ESM provider'
+
+providers:
+  - file://../providers/skill-metadata-esm.mjs
+
+prompts:
+  - 'Invoke a local skill'
+
+tests:
+  - assert:
+      - type: contains
+        value: 'Skill smoke:'
+      - type: skill-used
+        value: smoke-skill
+      - type: not-skill-used
+        value:
+          pattern: 'missing-*'

test/smoke/fixtures/configs/starts-with-assertion.yaml

@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - starts-with assertion (5.1.4)'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello, World!'
+
+tests:
+  - vars: {}
+    assert:
+      - type: starts-with
+        value: 'Hello'

test/smoke/fixtures/configs/tag-test.yaml

@@ -0,0 +1,24 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Smoke test - CLI tag overrides'
+
+tags:
+  build: config-build
+  team: evals
+
+commandLineOptions:
+  tags:
+    build: config-default-build
+    source: smoke-default
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello {{name}}'
+
+tests:
+  - vars:
+      name: World
+    assert:
+      - type: contains
+        value: Hello

test/smoke/fixtures/configs/test-threshold.yaml

@@ -0,0 +1,21 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Config for testing threshold option (minimum score for test to pass)
+description: 'Smoke test - test threshold'
+
+providers:
+  - echo
+
+prompts:
+  - 'Hello World'
+
+tests:
+  # Test with threshold - requires 50% of assertions to pass
+  - threshold: 0.5
+    assert:
+      - type: contains
+        value: Hello
+      - type: contains
+        value: World
+      # This will fail, but overall test passes due to threshold
+      - type: contains
+        value: Nonexistent

test/smoke/fixtures/configs/tests-external.yaml

@@ -0,0 +1,11 @@
+- vars:
+    name: Henry
+  assert:
+    - type: contains
+      value: Henry
+
+- vars:
+    name: Ivy
+  assert:
+    - type: contains
+      value: Ivy

test/smoke/fixtures/configs/tests.csv

@@ -0,0 +1,3 @@
+name,age,__expected
+Alice,30,contains: Alice
+Bob,25,contains: Bob

test/smoke/fixtures/configs/tests.json

@@ -0,0 +1,10 @@
+[
+  {
+    "vars": { "name": "Charlie" },
+    "assert": [{ "type": "contains", "value": "Charlie" }]
+  },
+  {
+    "vars": { "name": "Diana" },
+    "assert": [{ "type": "contains", "value": "Diana" }]
+  }
+]