chore: import upstream snapshot with attribution

examples/openai-structured-output/README.md

@@ -0,0 +1,45 @@
+# openai-structured-output (OpenAI Structured Output Example)
+
+This example demonstrates how to define JSON schemas for OpenAI's Structured Output feature in two different ways:
+
+1. **Inline schema definition** - defined directly in the config file
+2. **External schema file** - stored in a separate JSON or YAML file and referenced with `file://`
+
+## Usage
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example openai-structured-output
+cd openai-structured-output
+```
+
+## Environment Variables
+
+This example requires:
+
+- `OPENAI_API_KEY` - Your OpenAI API key
+
+## Example Structure
+
+This example includes several files that demonstrate different approaches:
+
+| File                             | Description                                                 |
+| -------------------------------- | ----------------------------------------------------------- |
+| `promptfooconfig.chat.yaml`      | Chat API config using both inline and external schemas      |
+| `promptfooconfig.responses.yaml` | Responses API config using both inline and external schemas |
+| `schema.responses.yaml`          | External schema file for Responses API                      |
+| `schema.chat.json`               | External schema file for Chat API                           |
+
+## Running the Example
+
+```bash
+cd openai-structured-output
+promptfoo eval -c promptfooconfig.chat.yaml
+promptfoo eval -c promptfooconfig.responses.yaml
+```
+
+## Additional Resources
+
+- [OpenAI Structured Output Announcement](https://openai.com/index/introducing-structured-outputs-in-the-api/)
+- [promptfoo Documentation](https://promptfoo.dev)

examples/openai-structured-output/per-test-schema.yaml

@@ -0,0 +1,66 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+
+# Per-test structured output example
+# Demonstrates using different JSON schemas for different tests with the same prompt
+#
+# This example shows how to vary the structured output schema on a per-test basis,
+# allowing you to use a single prompt with different response formats.
+
+description: 'Per-test structured output with different JSON schemas'
+
+prompts:
+  # Single prompt used for all tests
+  - 'Answer this question: {{question}}'
+
+providers:
+  - id: openai:gpt-4o-mini
+    config:
+      temperature: 0
+
+# Parse JSON output so assertions can access properties directly
+defaultTest:
+  options:
+    transform: JSON.parse(output)
+
+tests:
+  # Test 1: Math problem - expects structured math response
+  - vars:
+      question: 'What is 15 * 7?'
+    options:
+      # Load complete response_format from external file
+      response_format: file://./schemas/math-response-format.json
+    assert:
+      - type: javascript
+        value: output.answer === 105
+      - type: javascript
+        value: typeof output.explanation === 'string'
+
+  # Test 2: Another math problem with same schema
+  - vars:
+      question: 'What is 23 + 19?'
+    options:
+      response_format: file://./schemas/math-response-format.json
+    assert:
+      - type: javascript
+        value: output.answer === 42
+
+  # Test 3: Comparison question - different schema!
+  - vars:
+      question: 'Compare apples and oranges as fruits'
+    options:
+      # Different schema for comparison responses
+      response_format: file://./schemas/comparison-response-format.json
+    assert:
+      - type: javascript
+        value: output.winner === 'item1' || output.winner === 'item2' || output.winner === 'tie'
+      - type: javascript
+        value: output.reasoning.length > 20
+
+  # Test 4: Another comparison with same schema
+  - vars:
+      question: 'Compare cats and dogs as pets'
+    options:
+      response_format: file://./schemas/comparison-response-format.json
+    assert:
+      - type: javascript
+        value: output.winner === 'item1' || output.winner === 'item2' || output.winner === 'tie'

examples/openai-structured-output/promptfooconfig.chat.yaml

@@ -0,0 +1,170 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'OpenAI Chat API with External and Inline Schema Examples'
+prompts:
+  - 'Analyze the following customer support query: "{{query}}" and respond with structured JSON data.'
+
+providers:
+  # GPT-5 models with structured output
+  - id: openai:chat:gpt-5.4-nano
+    label: GPT-5.4 Nano External Schema
+    config:
+      max_completion_tokens: 800
+      seed: 123456
+      response_format: file://schema.chat.json # External schema file reference
+
+  - id: openai:chat:gpt-5.4-mini
+    label: GPT-5.4 Mini Inline Schema
+    config:
+      max_completion_tokens: 1000
+      response_format:
+        type: json_schema
+        json_schema:
+          name: enhanced_customer_support_analysis
+          strict: true
+          schema:
+            type: object
+            properties:
+              query_summary:
+                type: string
+                description: "A brief summary of the customer's query"
+              category:
+                type: string
+                enum:
+                  [
+                    'billing',
+                    'technical_issue',
+                    'product_inquiry',
+                    'complaint',
+                    'feature_request',
+                    'feedback',
+                    'other',
+                  ]
+                description: "The main category of the customer's query"
+              sentiment:
+                type: string
+                enum: ['very_positive', 'positive', 'neutral', 'negative', 'very_negative']
+                description: "The overall sentiment of the customer's query"
+              urgency:
+                type: string
+                enum: ['1', '2', '3', '4', '5']
+                description: 'The urgency level of the query, where 1 is lowest and 5 is highest'
+              suggested_actions:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    action:
+                      type: string
+                      description: 'A specific action to be taken'
+                    priority:
+                      type: string
+                      enum: ['low', 'medium', 'high', 'critical']
+                    estimated_time:
+                      type: string
+                      description: 'Estimated time to complete this action'
+                  required: ['action', 'priority', 'estimated_time']
+                  additionalProperties: false
+              estimated_resolution_time:
+                type: string
+                description: "Estimated time to resolve the query (e.g., '2 hours', '1 day')"
+              follow_up_required:
+                type: boolean
+                description: 'Whether a follow-up is required after initial resolution'
+            required:
+              [
+                'query_summary',
+                'category',
+                'sentiment',
+                'urgency',
+                'suggested_actions',
+                'estimated_resolution_time',
+                'follow_up_required',
+              ]
+            additionalProperties: false
+
+  # Legacy models for comparison
+  - id: openai:chat:gpt-4.1-nano
+    label: GPT-4.1 Nano External Schema
+    config:
+      temperature: 0.2
+      max_tokens: 800
+      seed: 123456
+      response_format: file://schema.chat.json # External schema file reference
+
+  - id: openai:chat:gpt-4.1-mini
+    label: GPT-4.1 Mini Inline Schema
+    config:
+      temperature: 0.3
+      max_tokens: 1000
+      response_format:
+        type: json_schema
+        json_schema:
+          name: enhanced_customer_support_analysis
+          strict: true
+          schema:
+            type: object
+            properties:
+              query_summary:
+                type: string
+                description: "A brief summary of the customer's query"
+              category:
+                type: string
+                enum:
+                  [
+                    'billing',
+                    'technical_issue',
+                    'product_inquiry',
+                    'complaint',
+                    'feature_request',
+                    'feedback',
+                    'other',
+                  ]
+                description: "The main category of the customer's query"
+              sentiment:
+                type: string
+                enum: ['very_positive', 'positive', 'neutral', 'negative', 'very_negative']
+                description: "The overall sentiment of the customer's query"
+              urgency:
+                type: string
+                enum: ['1', '2', '3', '4', '5']
+                description: 'The urgency level of the query, where 1 is lowest and 5 is highest'
+              suggested_actions:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    action:
+                      type: string
+                      description: 'A specific action to be taken'
+                    priority:
+                      type: string
+                      enum: ['low', 'medium', 'high', 'critical']
+                    estimated_time:
+                      type: string
+                      description: 'Estimated time to complete this action'
+                  required: ['action', 'priority', 'estimated_time']
+                  additionalProperties: false
+              estimated_resolution_time:
+                type: string
+                description: "Estimated time to resolve the query (e.g., '2 hours', '1 day')"
+              follow_up_required:
+                type: boolean
+                description: 'Whether a follow-up is required after initial resolution'
+            required:
+              [
+                'query_summary',
+                'category',
+                'sentiment',
+                'urgency',
+                'suggested_actions',
+                'estimated_resolution_time',
+                'follow_up_required',
+              ]
+            additionalProperties: false
+
+tests:
+  - vars:
+      query: "I've been charged twice for my subscription this month. Can you please refund the extra charge?"
+
+  - vars:
+      query: "How do I change my password? I can't find the option in my account settings."

examples/openai-structured-output/promptfooconfig.responses.yaml

@@ -0,0 +1,79 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'OpenAI Responses API with External Schema File Example'
+prompts:
+  - 'Analyze the following customer support query: "{{query}}" and respond with structured JSON data.'
+
+providers:
+  - id: openai:responses:gpt-4.1-nano
+    config:
+      temperature: 0.3
+      max_output_tokens: 600
+      top_p: 0.95
+      instructions: 'You are an expert customer support analyst. Analyze the customer query and provide structured information following the exact JSON schema provided.'
+      response_format: file://schema.responses.yaml # External schema file reference
+
+  - id: openai:responses:gpt-4.1-mini
+    label: Responses API (Inline Schema)
+    config:
+      temperature: 0.2
+      max_output_tokens: 800
+      instructions: 'You are an expert customer support analyst. Analyze the customer query and provide structured information following the exact JSON schema provided.'
+      response_format:
+        type: object
+        properties:
+          query_summary:
+            type: string
+            description: "A brief summary of the customer's query"
+          category:
+            type: string
+            enum:
+              [
+                'billing',
+                'technical_issue',
+                'product_inquiry',
+                'complaint',
+                'feature_request',
+                'other',
+              ]
+            description: "The main category of the customer's query"
+          sentiment:
+            type: string
+            enum: ['positive', 'neutral', 'negative']
+            description: "The overall sentiment of the customer's query"
+          urgency:
+            type: string
+            enum: ['1', '2', '3', '4', '5']
+            description: 'The urgency level of the query, where 1 is lowest and 5 is highest'
+          suggested_actions:
+            type: array
+            items:
+              type: object
+              properties:
+                action:
+                  type: string
+                  description: 'A specific action to be taken'
+                priority:
+                  type: string
+                  enum: ['low', 'medium', 'high']
+              required: ['action', 'priority']
+              additionalProperties: false
+          estimated_resolution_time:
+            type: string
+            description: "Estimated time to resolve the query (e.g., '2 hours', '1 day')"
+        required:
+          [
+            'query_summary',
+            'category',
+            'sentiment',
+            'urgency',
+            'suggested_actions',
+            'estimated_resolution_time',
+          ]
+        additionalProperties: false
+
+tests:
+  - vars:
+      query: "I've been charged twice for my subscription this month. Can you please refund the extra charge?"
+
+  - vars:
+      query: "How do I change my password? I can't find the option in my account settings."

examples/openai-structured-output/schema.chat.json

@@ -0,0 +1,69 @@
+{
+  "type": "json_schema",
+  "json_schema": {
+    "name": "customer_support_analysis",
+    "strict": true,
+    "schema": {
+      "type": "object",
+      "properties": {
+        "query_summary": {
+          "type": "string",
+          "description": "A brief summary of the customer's query"
+        },
+        "category": {
+          "type": "string",
+          "enum": [
+            "billing",
+            "technical_issue",
+            "product_inquiry",
+            "complaint",
+            "feature_request",
+            "other"
+          ],
+          "description": "The main category of the customer's query"
+        },
+        "sentiment": {
+          "type": "string",
+          "enum": ["positive", "neutral", "negative"],
+          "description": "The overall sentiment of the customer's query"
+        },
+        "urgency": {
+          "type": "string",
+          "enum": ["1", "2", "3", "4", "5"],
+          "description": "The urgency level of the query, where 1 is lowest and 5 is highest"
+        },
+        "suggested_actions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "action": {
+                "type": "string",
+                "description": "A specific action to be taken"
+              },
+              "priority": {
+                "type": "string",
+                "enum": ["low", "medium", "high"]
+              }
+            },
+            "required": ["action", "priority"],
+            "additionalProperties": false
+          }
+        },
+        "estimated_resolution_time": {
+          "type": "string",
+          "description": "Estimated time to resolve the query (e.g., '2 hours', '1 day')"
+        }
+      },
+      "required": [
+        "query_summary",
+        "category",
+        "sentiment",
+        "urgency",
+        "suggested_actions",
+        "estimated_resolution_time"
+      ],
+      "additionalProperties": false
+    }
+  }
+}

examples/openai-structured-output/schema.responses.yaml

@@ -0,0 +1,43 @@
+type: object
+properties:
+  query_summary:
+    type: string
+    description: "A brief summary of the customer's query"
+  category:
+    type: string
+    enum: ['billing', 'technical_issue', 'product_inquiry', 'complaint', 'feature_request', 'other']
+    description: "The main category of the customer's query"
+  sentiment:
+    type: string
+    enum: ['positive', 'neutral', 'negative']
+    description: "The overall sentiment of the customer's query"
+  urgency:
+    type: string
+    enum: ['1', '2', '3', '4', '5']
+    description: 'The urgency level of the query, where 1 is lowest and 5 is highest'
+  suggested_actions:
+    type: array
+    items:
+      type: object
+      properties:
+        action:
+          type: string
+          description: 'A specific action to be taken'
+        priority:
+          type: string
+          enum: ['low', 'medium', 'high']
+      required: ['action', 'priority']
+      additionalProperties: false
+  estimated_resolution_time:
+    type: string
+    description: "Estimated time to resolve the query (e.g., '2 hours', '1 day')"
+required:
+  [
+    'query_summary',
+    'category',
+    'sentiment',
+    'urgency',
+    'suggested_actions',
+    'estimated_resolution_time',
+  ]
+additionalProperties: false

examples/openai-structured-output/schemas/comparison-response-format.json

@@ -0,0 +1,31 @@
+{
+  "type": "json_schema",
+  "json_schema": {
+    "name": "comparison_response",
+    "strict": true,
+    "schema": {
+      "type": "object",
+      "properties": {
+        "item1": {
+          "type": "string",
+          "description": "First item being compared"
+        },
+        "item2": {
+          "type": "string",
+          "description": "Second item being compared"
+        },
+        "winner": {
+          "type": "string",
+          "enum": ["item1", "item2", "tie"],
+          "description": "Which item is better or if they're equal"
+        },
+        "reasoning": {
+          "type": "string",
+          "description": "Explanation of the comparison and decision"
+        }
+      },
+      "required": ["item1", "item2", "winner", "reasoning"],
+      "additionalProperties": false
+    }
+  }
+}

examples/openai-structured-output/schemas/math-response-format.json

@@ -0,0 +1,22 @@
+{
+  "type": "json_schema",
+  "json_schema": {
+    "name": "math_response",
+    "strict": true,
+    "schema": {
+      "type": "object",
+      "properties": {
+        "answer": {
+          "type": "number",
+          "description": "The numerical answer to the problem"
+        },
+        "explanation": {
+          "type": "string",
+          "description": "Step-by-step explanation of the solution"
+        }
+      },
+      "required": ["answer", "explanation"],
+      "additionalProperties": false
+    }
+  }
+}