chore: import upstream snapshot with attribution

examples/config-extension-api/README.md

@@ -0,0 +1,71 @@
+# config-extension-api (Custom Extensions Example for promptfoo)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example config-extension-api
+cd config-extension-api
+```
+
+This example demonstrates how to leverage promptfoo's powerful extensions API to implement custom setup and teardown hooks for individual tests. These extensions can be defined in either Python or JavaScript, providing flexibility for your preferred programming environment.
+
+## Extension Hooks Overview
+
+promptfoo supports four types of extension hooks, each triggered at a specific point in the evaluation lifecycle:
+
+1. `beforeAll`: Executed once before the entire test suite begins
+2. `afterAll`: Executed once after the entire test suite completes
+3. `beforeEach`: Executed before each individual test
+4. `afterEach`: Executed after each individual test
+
+Each hook receives a `hookName` parameter and a `context` object containing relevant data for that specific hook type.
+
+The `beforeAll` and `beforeEach` hooks should return the `context` object, while the `afterAll` and `afterEach` hooks should not return anything.
+
+For comprehensive information on implementing and using these hooks, refer to the [Extension Hooks](https://www.promptfoo.dev/docs/configuration/reference/#extension-hooks) section in the official documentation.
+
+## Configuring Extensions
+
+Specify your extensions in the `promptfooconfig.yaml` file:
+
+```yaml
+extensions:
+  - file://path/to/your/extension.py:extension_hook
+  - file://path/to/your/extension.js:extensionHook
+```
+
+Note: You must include the function name after the file path, separated by a colon (`:`).
+
+## Why Use Extensions?
+
+Extensions in promptfoo empower you to:
+
+1. Enhance test functionality with custom pre- and post-test actions
+2. Dynamically prepare and clean up test environments
+3. Seamlessly integrate with external systems or services for comprehensive testing
+4. Implement custom logging or reporting mechanisms
+
+These extensions function similarly to "before" and "after" hooks in popular unit testing frameworks, allowing you to set up and tear down test environments with precision and control.
+
+## Practical Use Cases for Extensions
+
+1. **Environment Setup/Teardown**: Dynamically create and remove files or resources for each test.
+2. **AI Agent Testing**: Configure complex scenarios, such as deploying Kubernetes manifests before evaluation.
+3. **Model Loading**: Efficiently load ML models into memory before tests and unload them afterward.
+4. **Local Server Management**: Programmatically start and stop local language model servers (e.g., llama.cpp, ollama) for testing.
+5. **Connection Management**: Automate the opening and closing of connection tunnels or database connections.
+6. **Credential Handling**: Securely load and manage credentials or authentication tokens for testing environments.
+
+## Running the Example
+
+To execute this example, use the following command in your terminal:
+
+```sh
+LOG_LEVEL=debug promptfoo eval
+```
+
+Then
+
+```sh
+promptfoo view
+```

examples/config-extension-api/hooks.js

@@ -0,0 +1,79 @@
+/**
+ * WARNING: The `counter` variable is NOT thread-safe. When running with concurrency > 1,
+ * the counter may not accurately track test numbers. Consider using per-test metadata
+ * for accurate tracking in concurrent scenarios.
+ */
+let counter = 0;
+
+/**
+ * Handles different extension hooks for promptfoo.
+ *
+ * IMPORTANT: The function signature is (hookName, context) where:
+ * - hookName: 'beforeAll' | 'beforeEach' | 'afterEach' | 'afterAll'
+ * - context: The hook-specific data (e.g., { suite } for beforeAll)
+ *
+ * @param {string} hookName - The name of the hook being called.
+ * @param {Object} context - A dictionary containing contextual information for the hook.
+ *   The contents vary depending on the hook being called:
+ *   - beforeAll: { suite: TestSuite }
+ *   - beforeEach: { test: TestCase }
+ *   - afterEach: { test: TestCase, result: EvaluateResult }
+ *   - afterAll: { results: EvaluateResult[], suite: TestSuite, ... }
+ * @returns {Object|undefined} The "beforeAll" and "beforeEach" hooks should return the context object,
+ *   while the "afterAll" and "afterEach" hooks should not return anything.
+ */
+async function extensionHook(hookName, context) {
+  if (hookName === 'beforeAll') {
+    console.log(`Setting up test suite: ${context.suite.description || 'Unnamed suite'}`);
+    console.log(`Total prompts: ${context.suite.prompts?.length || 0}`);
+    console.log(`Total providers: ${context.suite.providers?.length || 0}`);
+    console.log(`Total tests: ${context.suite.tests?.length || 0}`);
+
+    // Add an additional test case to the suite:
+    context.suite.tests.push({
+      vars: {
+        body: "It's a beautiful day",
+        language: 'Spanish',
+      },
+      assert: [{ type: 'contains', value: 'Es un día hermoso.' }],
+    });
+
+    // Add an additional default assertion to the suite:
+    context.suite.defaultTest.assert.push({ type: 'is-json' });
+
+    return context;
+  } else if (hookName === 'beforeEach') {
+    console.log(`Preparing test`);
+
+    // All languages are now pirate:
+    context.test.vars.language = `Pirate ${context.test.vars.language}`;
+
+    return context;
+  } else if (hookName === 'afterEach') {
+    console.log(
+      `Completed test ${counter++}${context.result ? `, Result: ${context.result.success ? 'Pass' : 'Fail'}, Score: ${context.result.score}` : ''}`,
+    );
+
+    // Access sessionId if available (from multi-turn conversations or stateful tests)
+    if (context.result?.metadata?.sessionId) {
+      console.log(`Session ID: ${context.result.metadata.sessionId}`);
+    }
+    if (context.result?.metadata?.sessionIds) {
+      console.log(`Session IDs: ${context.result.metadata.sessionIds}`);
+    }
+  } else if (hookName === 'afterAll') {
+    console.log('Test suite completed');
+    console.log(`Total tests run: ${context.results?.length || 0}`);
+
+    const successes = context.results?.filter((r) => r.success).length || 0;
+    const failures = context.results?.filter((r) => !r.success).length || 0;
+    console.log(`Successes: ${successes}`);
+    console.log(`Failures: ${failures}`);
+
+    const totalTokenUsage =
+      context.results?.reduce((sum, r) => sum + (r.response?.tokenUsage?.total || 0), 0) || 0;
+    console.log(`Total token usage: ${totalTokenUsage}`);
+  }
+}
+
+module.exports = extensionHook;

examples/config-extension-api/hooks.py

@@ -0,0 +1,139 @@
+"""Extension hooks for promptfoo.
+
+This module provides functionality for handling extension hooks in promptfoo.
+It allows for executing custom actions before and after test suites and
+individual evaluations, as well as running setup and teardown commands.
+
+IMPORTANT: The function signature is (hook_name, context) where:
+- hook_name: 'beforeAll', 'beforeEach', 'afterEach', or 'afterAll'
+- context: The hook-specific data (e.g., {'suite': ...} for beforeAll)
+
+WARNING: The `counter` variable is NOT thread-safe. When running with concurrency > 1,
+the counter may not accurately track test numbers. Consider using thread-safe counters
+or per-test metadata for accurate tracking in concurrent scenarios.
+"""
+
+import logging
+import os
+from datetime import datetime
+from typing import Optional
+
+# Set up logging only if it hasn't been set up already
+if not logging.getLogger().handlers:
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    log_filename = f"promptfoo_run_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+    log_path = os.path.join(current_dir, log_filename)
+    logging.basicConfig(
+        filename=log_path,
+        level=logging.INFO,
+        format="%(asctime)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+# WARNING: Not thread-safe with concurrency > 1
+counter = 0
+
+
+def extension_hook(hook_name: str, context: dict) -> Optional[dict]:
+    """Handles different extension hooks for promptfoo.
+
+    This function is called at various points during the test execution process.
+    It logs information about the test suite and individual tests.
+
+    Args:
+        hook_name (str): The name of the hook being called.
+            One of: 'beforeAll', 'beforeEach', 'afterEach', 'afterAll'
+        context (dict): A dictionary containing contextual information for the hook.
+            The contents vary depending on the hook being called:
+            - beforeAll: {'suite': TestSuite}
+            - beforeEach: {'test': TestCase}
+            - afterEach: {'test': TestCase, 'result': EvaluateResult}
+            - afterAll: {'results': EvaluateResult[], 'suite': TestSuite, ...}
+
+    Returns:
+        context (Optional[dict]): The "beforeAll" and "beforeEach" hooks should return
+            the context object (potentially modified), while the "afterAll" and "afterEach"
+            hooks should not return anything.
+
+    Global Variables:
+        counter (int): Keeps track of the number of tests completed.
+            WARNING: Not thread-safe when running with concurrency > 1.
+
+    Logs:
+        Information about the test suite and individual tests, including setup,
+        completion, results, and token usage.
+    """
+    global counter
+
+    if hook_name == "beforeAll":
+        suite = context.get("suite", {})
+        logging.info(
+            f"Setting up test suite: {suite.get('description') or 'Unnamed suite'}"
+        )
+        logging.info(f"Total prompts: {len(suite.get('prompts', []))}")
+        logging.info(f"Total providers: {len(suite.get('providers', []))}")
+        logging.info(f"Total tests: {len(suite.get('tests', []))}")
+
+        # Add an additional test case to the suite:
+        context["suite"]["tests"].append(
+            {
+                "vars": {
+                    "body": "It's a beautiful day",
+                    "language": "Spanish",
+                },
+                "assert": [{"type": "contains", "value": "Es un día hermoso."}],
+            }
+        )
+
+        # Add an additional default assertion to the suite:
+        context["suite"]["defaultTest"]["assert"].append({"type": "is-json"})
+
+        return context
+
+    elif hook_name == "beforeEach":
+        logging.info("Preparing test")
+
+        # All languages are now pirate:
+        context["test"]["vars"]["language"] = (
+            f"Pirate {context['test']['vars']['language']}"
+        )
+
+        return context
+
+    elif hook_name == "afterEach":
+        result = context.get("result", {})
+        result_str = ""
+        if result:
+            success = "Pass" if result.get("success") else "Fail"
+            score = result.get("score", 0)
+            result_str = f", Result: {success}, Score: {score}"
+        logging.info(f"Completed test {counter}{result_str}")
+
+        # Access sessionId if available (from multi-turn conversations or stateful tests)
+        session_id = result.get("metadata", {}).get("sessionId")
+        if session_id:
+            logging.info(f"Session ID: {session_id}")
+
+        session_ids = result.get("metadata", {}).get("sessionIds")
+        if session_ids:
+            logging.info(f"Session IDs: {session_ids}")
+
+        counter += 1
+
+    elif hook_name == "afterAll":
+        results = context.get("results", [])
+        logging.info("Test suite completed")
+        logging.info(f"Total tests run: {len(results)}")
+
+        successes = sum(1 for r in results if r.get("success"))
+        failures = sum(1 for r in results if not r.get("success"))
+        logging.info(f"Successes: {successes}")
+        logging.info(f"Failures: {failures}")
+
+        total_token_usage = sum(
+            r.get("response", {}).get("tokenUsage", {}).get("total", 0) for r in results
+        )
+        logging.info(f"Total token usage: {total_token_usage}")
+
+    logging.info("")  # Add a blank line for readability between hooks
+    return None

examples/config-extension-api/promptfooconfig.yaml

@@ -0,0 +1,31 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Extension Hook Example
+
+prompts:
+  - 'Rephrase this in {{language}}: {{body}}'
+
+providers:
+  - openai:gpt-4.1-mini
+
+extensions:
+  - file://hooks.py:extension_hook
+  # - file://hooks.js:extensionHook
+
+tests:
+  - vars:
+      body: 'Hello, how are you?'
+      language: 'Norwegian'
+    assert:
+      - type: contains-any
+        value:
+          - 'Hallo'
+          - 'Hei'
+          - 'Hå'
+      - type: not-contains
+        value: 'Hello'
+  - vars:
+      body: 'I love sailing on the sea.'
+      language: 'French'
+    assert:
+      - type: llm-rubric
+        value: 'The response is in French'