chore: import upstream snapshot with attribution

examples/provider-python/.gitignore

@@ -0,0 +1,2 @@
+venv
+__pycache__

examples/provider-python/README.md

@@ -0,0 +1,99 @@
+# provider-python (Python Provider)
+
+This example demonstrates how to create a custom Python provider for promptfoo that integrates with the OpenAI API.
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example provider-python
+cd provider-python
+```
+
+## Overview
+
+The Python provider allows you to use Python code as a provider in promptfoo evaluations. This is useful when you need to:
+
+1. Call APIs from Python libraries
+2. Implement custom logic before or after calling LLMs
+3. Process responses in specific ways
+4. Track token usage and other metrics
+
+## Environment Variables
+
+This example requires the following environment variable:
+
+- `OPENAI_API_KEY` - Your OpenAI API key
+
+You can set this in a `.env` file or directly in your environment.
+
+## Requirements
+
+- Python with the OpenAI package installed (`pip install openai`)
+
+## Files
+
+- `provider.py` - The Python provider implementation that calls OpenAI's API
+- `promptfooconfig.yaml` - Configuration for promptfoo evaluation with proper YAML schema reference
+- `configs/` directory:
+  - `fileConfig.yaml` - YAML configuration for model settings
+  - `fileConfig.js` - JavaScript configuration for formatting options
+  - `fileConfig.py` - Python configuration for additional parameters
+
+## Implementation Details
+
+The Python provider is defined in `provider.py` and includes:
+
+1. A `call_api` function that makes API calls to OpenAI
+2. Token usage extraction from the API response
+3. Multiple sample functions showing different ways to call the API
+
+By default, the example is configured to use `gpt-4.1-mini` model, but you can modify it to use other models as needed.
+
+## Expected Output
+
+When you run this example, you'll see:
+
+1. The prompts being submitted to your Python provider
+2. Responses from the OpenAI API
+3. Token usage statistics for each completion
+4. Evaluation results in a table format
+
+## File Reference Configuration
+
+The example demonstrates how to load configuration values from external files using the `file://` protocol directly in the `promptfooconfig.yaml` file. It shows three main file types:
+
+1. **YAML file** (`configs/fileConfig.yaml`): Contains model settings like temperature and max tokens
+2. **JavaScript file** (`configs/fileConfig.js`): Provides formatting options through a function export
+3. **Python file** (`configs/fileConfig.py`): Supplies additional parameters through a Python function
+
+The provider supports loading from:
+
+- JSON files (`.json`)
+- YAML files (`.yaml`, `.yml`)
+- JavaScript files (`.js`, `.mjs`, `.ts`, `.cjs`)
+- Python files (`.py`)
+- Text files (`.txt`, `.md`)
+
+You can see how this works in the `promptfooconfig.yaml` file:
+
+```yaml
+providers:
+  - id: 'file://provider.py:call_api'
+    config:
+      # YAML
+      settings: 'file://configs/fileConfig.yaml'
+      # JavaScript file
+      formatting: 'file://configs/fileConfig.js:getFormatConfig'
+      nested: # Python file
+        parameters: 'file://configs/fileConfig.py:get_params'
+```
+
+Run the example with:
+
+```bash
+npx promptfoo@latest evaluate -c examples/provider-python/promptfooconfig.yaml
+```
+
+## Learn More
+
+For more information on creating custom providers, see the [promptfoo documentation](https://promptfoo.dev/docs/providers/python/).

examples/provider-python/configs/fileConfig.js

@@ -0,0 +1,10 @@
+/**
+ * Simple formatting configuration - this is here to demonstrate how to load formatting configuration from a JavaScript file
+ */
+export function getFormatConfig() {
+  return {
+    uppercase: false,
+    prefix: 'Question:',
+    timestamp: new Date().toISOString(),
+  };
+}

examples/provider-python/configs/fileConfig.py

@@ -0,0 +1,19 @@
+"""
+Simple parameters configuration - this is here to demonstrate how to load parameters configuration from a Python file
+"""
+
+
+def get_params():
+    """
+    Returns basic parameters for the API calls
+
+    Returns:
+        dict: A dictionary containing parameters
+    """
+    return {
+        "max_tokens": 1000,
+        "frequency_penalty": 0,
+        "presence_penalty": 0,
+        "timeout": 30,
+        "foo": "bar",
+    }

examples/provider-python/configs/fileConfig.yaml

@@ -0,0 +1,6 @@
+# Simple settings for Python provider - these are here to demonstrate how to load settings
+# from a YAML file and not used to configure the provider.
+model: 'gpt-4.1-mini'
+temperature: 0.7
+log_level: DEBUG
+max_tokens: 500

examples/provider-python/promptfooconfig.yaml

@@ -0,0 +1,42 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Custom Python provider with functions and multi-format configs
+
+prompts:
+  - 'Write a very concise funny tweet about {{topic}}'
+
+providers:
+  - id: file://provider.py # defaults to call_api function
+    config:
+      someOption: foobar
+
+  - id: file://provider.py:some_other_function
+
+  - id: file://provider.py:async_provider
+    label: async provider
+
+  # Demonstrates how to load configuration from various file formats
+  - id: 'file://provider.py:call_api'
+    config:
+      # YAML
+      settings: 'file://configs/fileConfig.yaml'
+      # JavaScript file
+      formatting: 'file://configs/fileConfig.js:getFormatConfig'
+      nested: # Python file
+        parameters: 'file://configs/fileConfig.py:get_params'
+
+tests:
+  - vars:
+      topic: bananas
+    assert:
+      - type: contains
+        value: Bananamax
+  - vars:
+      topic: fruits
+    assert:
+      - type: llm-rubric
+        value: includes at least one emoji
+  - vars:
+      topic: turtles
+    assert:
+      - type: llm-rubric
+        value: is funny

examples/provider-python/provider.py

@@ -0,0 +1,83 @@
+from openai import AsyncOpenAI, OpenAI
+
+async_client = AsyncOpenAI()
+client = OpenAI()
+
+
+def call_api(prompt, options, context):
+    # Get config values
+    # some_option = options.get("config").get("someOption")
+
+    chat_completion = client.chat.completions.create(
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a marketer working for a startup called Bananamax.",
+            },
+            {
+                "role": "user",
+                "content": prompt,
+            },
+        ],
+        model="gpt-4.1-mini",
+    )
+
+    # Extract token usage information from the response
+    token_usage = None
+    if hasattr(chat_completion, "usage"):
+        token_usage = {
+            "total": chat_completion.usage.total_tokens,
+            "prompt": chat_completion.usage.prompt_tokens,
+            "completion": chat_completion.usage.completion_tokens,
+        }
+
+    return {
+        "output": chat_completion.choices[0].message.content,
+        "tokenUsage": token_usage,
+        "metadata": {
+            "config": options.get("config", {}),
+        },
+    }
+
+
+def some_other_function(prompt, options, context):
+    return call_api(prompt + "\nWrite in ALL CAPS", options, context)
+
+
+async def async_provider(prompt, options, context):
+    chat_completion = await async_client.chat.completions.create(
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a marketer working for a startup called Bananamax.",
+            },
+            {
+                "role": "user",
+                "content": prompt,
+            },
+        ],
+        model="gpt-4o",
+    )
+
+    # Extract token usage information from the async response
+    token_usage = None
+    if hasattr(chat_completion, "usage"):
+        token_usage = {
+            "total": chat_completion.usage.total_tokens,
+            "prompt": chat_completion.usage.prompt_tokens,
+            "completion": chat_completion.usage.completion_tokens,
+        }
+
+    return {
+        "output": chat_completion.choices[0].message.content,
+        "tokenUsage": token_usage,
+    }
+
+
+if __name__ == "__main__":
+    # Example usage showing prompt, options with config, and context with vars
+    prompt = "What is the weather in San Francisco?"
+    options = {"config": {"optionFromYaml": 123}}
+    context = {"vars": {"location": "San Francisco"}}
+
+    print(call_api(prompt, options, context))

examples/provider-python/requirements.txt

@@ -0,0 +1,10 @@
+annotated-types==0.6.0
+anyio==4.2.0
+certifi==2025.10.5
+distro==1.9.0
+exceptiongroup==1.2.0
+h11==0.16.0
+httpcore==1.0.2
+httpx==0.26.0
+idna==3.15
+openai==2.3.0