chore: import upstream snapshot with attribution

examples/config-custom-prompt-function/README.md

@@ -0,0 +1,84 @@
+# config-custom-prompt-function (Prompt Functions and Format Examples)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example config-custom-prompt-function
+cd config-custom-prompt-function
+```
+
+This example demonstrates the full range of prompt formats supported by promptfoo, with special focus on prompt functions that can return both content and configuration.
+
+## Quick Start
+
+```bash
+# Copy this example to your project
+npx promptfoo@latest init --example config-custom-prompt-function
+
+# Set required API keys
+export OPENAI_API_KEY=your_openai_key
+export ANTHROPIC_API_KEY=your_anthropic_key
+
+# Run the evaluation
+npx promptfoo@latest eval
+
+# View results
+npx promptfoo@latest view
+```
+
+## What This Example Shows
+
+### Supported Prompt Formats
+
+- Raw text prompts
+- File-based prompts (txt, yaml, json, jsonl, md, j2)
+- Glob patterns for multiple files
+- JSON chat formats
+- Markdown formatted prompts
+- Jinja2 template files
+
+### Prompt Functions
+
+- JavaScript prompt functions (CommonJS and ESM)
+- TypeScript prompt functions
+- Python prompt functions and class methods
+- Multiple functions in a single file
+- **Dynamic configuration** - returning `{ prompt, config }` objects that override provider settings
+
+## Example Structure
+
+- `prompt.txt`, `prompt.yaml`, etc. - Various prompt file formats
+- `prompt.j2` - Jinja2 template format
+- `prompt_chat.js/ts` - Chat format examples
+- `prompt_multiple.js` - Multiple functions in one file
+- `prompt_esm.mjs` - ESM module format
+- `prompt_python.py` - Python examples
+- `prompt_config.js/py` - Functions returning dynamic configuration
+- `subfolder/` - Demonstrates nested file structures
+- `promptfooconfig.yaml` - Complete configuration showing all prompt types
+
+## Configuration Features
+
+- Dynamic adjustment of model parameters (temperature, tokens) based on content
+- Content-aware JSON schema formatting
+- Configuration merging (function values override provider defaults)
+- Provider-specific prompt configurations
+- Multiple provider comparisons (OpenAI and Anthropic)
+
+## Customization
+
+To adapt this example:
+
+1. Explore different prompt formats to find what works best for your use case
+2. Use dynamic configurations to optimize model behavior based on input context
+3. Compare how different providers respond to the same prompts
+4. Create your own prompt functions for advanced use cases
+
+## Documentation
+
+Learn more about:
+
+- [Prompt Formats](https://www.promptfoo.dev/docs/configuration/prompts/#prompts)
+- [Prompt Functions](https://www.promptfoo.dev/docs/configuration/prompts/#prompt-functions)
+- [Provider Configuration](https://www.promptfoo.dev/docs/providers/)
+- [JSON Schema Response Formats](https://www.promptfoo.dev/docs/providers/openai/#json-mode-structured-outputs)

examples/config-custom-prompt-function/another_topic.txt

@@ -0,0 +1 @@
+bruce lee

examples/config-custom-prompt-function/prompt.j2

@@ -0,0 +1,2 @@
+You are a helpful but somewhat poetic assistant.
+Please provide information about {{ topic }} in the style of a haiku.

examples/config-custom-prompt-function/prompt.jsonl

@@ -0,0 +1 @@
+[{"role": "system", "content": "You're an angry pirate. Be concise and stay in character."}, {"role": "user", "content": "Tell me about {{topic}}"}]

examples/config-custom-prompt-function/prompt.md

@@ -0,0 +1,5 @@
+You're an angry pirate.
+
+Be concise and stay in character.
+
+Tell me about {{topic}}

examples/config-custom-prompt-function/prompt.txt

@@ -0,0 +1,2 @@
+You're an angry pirate. Be concise and stay in character. Tell me about {{topic}}
+---

examples/config-custom-prompt-function/prompt.yaml

@@ -0,0 +1,4 @@
+- role: system
+  content: "You're an angry pirate. Be concise and stay in character."
+- role: user
+  content: 'Tell me about {{topic}}'

examples/config-custom-prompt-function/prompt_chat.js

@@ -0,0 +1,13 @@
+// An example prompt function that returns a JSON OpenAI-like "chat" object.
+module.exports = async function ({ vars }) {
+  return [
+    {
+      role: 'system',
+      content: `You're an angry pirate. Be concise and stay in character.`,
+    },
+    {
+      role: 'user',
+      content: `Tell me about ${vars.topic}`,
+    },
+  ];
+};

examples/config-custom-prompt-function/prompt_chat.ts

@@ -0,0 +1,13 @@
+// An example prompt function that returns a JSON OpenAI-like "chat" object.
+export default async function ({ vars }: { vars: Record<string, string> }) {
+  return [
+    {
+      role: 'system',
+      content: `You're an angry typescript developer. Be concise and stay in character.`,
+    },
+    {
+      role: 'user',
+      content: `Tell me about ${vars.topic}`,
+    },
+  ];
+}

examples/config-custom-prompt-function/prompt_config.js

@@ -0,0 +1,30 @@
+// A prompt function that returns both prompt content and configuration
+module.exports.promptWithConfig = async function ({ vars, provider }) {
+  // Dynamic configuration based on the topic
+  const isComplexTopic = vars.topic === 'the Roman Empire' || vars.topic === 'bob dylan';
+  const temperature = isComplexTopic ? 0.9 : 0.5;
+  const maxTokens = isComplexTopic ? 150 : 75;
+
+  // Return both prompt and config
+  return {
+    prompt: [
+      {
+        role: 'system',
+        content: `You're a knowledgeable historian using a ${provider.label || provider.id} model. 
+                  Be factual and concise. Use at most ${maxTokens} tokens.`,
+      },
+      {
+        role: 'user',
+        content: `Tell me about ${vars.topic}`,
+      },
+    ],
+    config: {
+      temperature,
+      max_tokens: maxTokens,
+      top_p: 0.95,
+      response_format: {
+        type: 'text',
+      },
+    },
+  };
+};

examples/config-custom-prompt-function/prompt_config.py

@@ -0,0 +1,45 @@
+import json
+import sys
+
+
+def prompt_with_config(context):
+    """
+    A Python prompt function that returns both prompt content and configuration.
+    """
+    vars = context["vars"]
+    provider = context.get("provider", {})
+
+    # Dynamic configuration based on the topic
+    if vars["topic"] == "the Roman Empire" or vars["topic"] == "bob dylan":
+        # Complex topics need more elaboration
+        temperature = 0.8
+        max_tokens = 200
+    else:
+        # Simple topics can be more constrained
+        temperature = 0.4
+        max_tokens = 100
+
+    # Return structured object with both prompt and config
+    return {
+        "prompt": [
+            {
+                "role": "system",
+                "content": f"You are a Python expert assistant using {provider.get('id', 'an AI')} model. Be technical and precise.",
+            },
+            {
+                "role": "user",
+                "content": f"Explain {vars['topic']} as if it were a Python library. What would its main classes and methods be?",
+            },
+        ],
+        "config": {
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+            "presence_penalty": 0.1,
+            "frequency_penalty": 0.3,
+        },
+    }
+
+
+if __name__ == "__main__":
+    # When executed directly, run the prompt_with_config function
+    print(json.dumps(prompt_with_config(json.loads(sys.argv[1]))))

examples/config-custom-prompt-function/prompt_esm.mjs

@@ -0,0 +1,13 @@
+// An example prompt function that returns a JSON OpenAI-like "chat" object.
+export default async function ({ vars }) {
+  return [
+    {
+      role: 'system',
+      content: `You're an angry pirate. Be concise and stay in character.`,
+    },
+    {
+      role: 'user',
+      content: `Tell me about ${vars.topic}`,
+    },
+  ];
+}

examples/config-custom-prompt-function/prompt_multiple.js

@@ -0,0 +1,26 @@
+// An example prompt function that returns a JSON OpenAI-like "chat" object.
+module.exports.prompt1 = async function ({ vars }) {
+  return [
+    {
+      role: 'system',
+      content: `You're an angry pirate. Be concise and stay in character.`,
+    },
+    {
+      role: 'user',
+      content: `Tell me about ${vars.topic}`,
+    },
+  ];
+};
+
+module.exports.prompt2 = async function ({ vars }) {
+  return [
+    {
+      role: 'system',
+      content: `You do not answer questions. You only make wolf noises.`,
+    },
+    {
+      role: 'user',
+      content: `Tell me about ${vars.topic}`,
+    },
+  ];
+};

examples/config-custom-prompt-function/prompt_python.py

@@ -0,0 +1,42 @@
+import json
+import sys
+
+
+def prompt1(context):
+    return (
+        f"Write a one-sentence insult for anyone who likes {context['vars']['topic']}."
+    )
+
+
+def generate_prompt(context):
+    return f"Describe {context['vars']['topic']} concisely, comparing it to the Python programming language."
+
+
+class Prompt:
+    system_prompt = "You're an angry pirate. Be concise and stay in character."
+    user_prompt = "Tell me about {}"
+
+    @staticmethod
+    def prompt(context):
+        return [
+            {"role": "system", "content": Prompt.system_prompt},
+            {
+                "role": "user",
+                "content": Prompt.user_prompt.format(context["vars"]["topic"]),
+            },
+        ]
+
+    @staticmethod
+    def prompt_with_cot(context):
+        return [
+            {"role": "system", "content": Prompt.system_prompt},
+            {
+                "role": "user",
+                "content": Prompt.user_prompt.format(context["vars"]["topic"])
+                + "\nLet's think step by step.",
+            },
+        ]
+
+
+if __name__ == "__main__":
+    print(generate_prompt(json.loads(sys.argv[1])))

examples/config-custom-prompt-function/prompt_string.js

@@ -0,0 +1,4 @@
+// An example prompt function that returns a simple string.
+module.exports = async function ({ vars }) {
+  return `Imagine you're an angry pirate. Be concise and stay in character. Tell me about ${vars.topic}`;
+};

examples/config-custom-prompt-function/promptfooconfig.yaml

@@ -0,0 +1,99 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Dynamic prompt generation using JavaScript functions
+
+prompts:
+  - 'You are an angry pirate. Be concise and stay in character. Tell me about {{topic}}'
+  - id: 'You are an angry pirate. Be concise and stay in character. Tell me about {{topic}}'
+    label: text
+
+  - id: file://prompt.txt
+    label: prompt_txt
+  - file://prompt.txt
+
+  - id: '*.txt'
+    label: prompt_glob
+  - '*.txt'
+  - 'file://*.txt'
+
+  - id: file://prompt.yaml
+    label: prompt_yaml
+  - file://prompt.yaml
+
+  - id: file://subfolder/prompt.json
+    label: prompt_json
+  - file://subfolder/prompt.json
+
+  - file://subfolder/*.json
+
+  - id: file://prompt.md
+    label: markdown prompt
+
+  - id: file://prompt.j2
+    label: jinja2 template
+
+  - id: file://prompt.jsonl
+    label: prompt_jsonl
+  - file://prompt.jsonl
+
+  - id: file://prompt_chat.js
+    label: prompt_chat (js)
+  - file://prompt_chat.js
+
+  - id: file://prompt_chat.ts
+    label: prompt_chat (ts)
+  - file://prompt_chat.ts
+
+  - id: file://prompt_string.js
+    label: prompt_string
+  - file://prompt_string.js
+
+  - id: file://prompt_multiple.js:prompt2
+    label: prompt_multiple with prompt2
+  - file://prompt_multiple.js:prompt2
+
+  - id: file://prompt_esm.mjs
+    label: prompt_esm
+  - file://prompt_esm.mjs
+
+  - id: file://prompt_python.py
+    label: prompt_python
+  - file://prompt_python.py
+
+  - id: file://prompt_python.py:prompt1
+    label: prompt_python with prompt1
+  - file://prompt_python.py:prompt1
+
+  - id: file://prompt_python.py:Prompt.prompt
+    label: prompt_python_class
+  - file://prompt_python.py:Prompt.prompt_with_cot
+
+  - id: file://prompt_config.js:promptWithConfig
+    label: JS Prompt with Dynamic Config
+
+  - id: file://prompt_config.py:prompt_with_config
+    label: Python Prompt with Dynamic Config
+
+  - id: file://prompts.csv
+    label: CSV prompts
+
+providers:
+  # Use the echo provider only to see rendered prompts without LLM calls
+  - echo
+  # - id: anthropic:claude-sonnet-4-6
+  #   label: Claude Sonnet 4.6
+  # - id: openai:gpt-4.1-mini
+  #   label: GPT-4o Mini
+  # config below is overridden by prompt functions with config
+  # config:
+  #   temperature: 0.0 # Default temperature, will be overridden by prompt function
+  #   max_tokens: 50 # Default max_tokens, will be overridden by prompt function
+
+tests:
+  - vars:
+      topic: the weather
+  - vars:
+      topic: bob dylan
+  - vars:
+      topic: the Roman Empire
+  - vars:
+      topic: file://another_topic.txt

examples/config-custom-prompt-function/prompts.csv

@@ -0,0 +1,5 @@
+prompt,label
+"Tell me about {{topic}} in the style of a children's book","Children's Book Style"
+"Explain {{topic}} as if you're a professor giving a lecture","Academic Lecture"
+"Write a short poem about {{topic}}","Poetic Format"
+"What's the relationship between {{topic}} and technology?","Tech Analysis" 

examples/config-custom-prompt-function/subfolder/prompt.json

@@ -0,0 +1,10 @@
+[
+  {
+    "role": "system",
+    "content": "You're an angry pirate. Be concise and stay in character."
+  },
+  {
+    "role": "user",
+    "content": "Tell me about {{topic}}"
+  }
+]