chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,96 @@
+# config-structured-outputs (Multi-Provider Structured Outputs)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example config-structured-outputs
+cd config-structured-outputs
+```
+
+This example demonstrates how to enforce structured JSON outputs using schema validation across multiple AI providers:
+
+- **OpenAI** - using `response_format` with `json_schema`
+- **Azure OpenAI** - using `response_format` with `json_schema`
+- **Anthropic** - using `output_format` with `json_schema`
+
+The example includes two prompt configurations for solving quirky math problems:
+
+- One that requires step-by-step problem solving (with `steps` array)
+- One that only requires the final answer (without `steps` array)
+
+## Environment Variables
+
+This example requires at least one of the following API keys:
+
+- `OPENAI_API_KEY` - Your OpenAI API key for testing with GPT models
+- `ANTHROPIC_API_KEY` - Your Anthropic API key for testing with Claude models
+- For Azure OpenAI: See [Azure OpenAI provider docs](https://promptfoo.dev/docs/providers/azure) for required credentials
+
+You can set these in a `.env` file or directly in your environment:
+
+```bash
+export OPENAI_API_KEY=your_api_key_here
+export ANTHROPIC_API_KEY=your_api_key_here
+```
+
+## Provider Differences
+
+### OpenAI/Azure Format
+
+OpenAI and Azure use `response_format` at the prompt level:
+
+```yaml
+prompts:
+  - raw: 'Your prompt here'
+    config:
+      response_format:
+        type: json_schema
+        json_schema:
+          name: schema_name
+          strict: true
+          schema:
+            # Your schema here
+```
+
+### Anthropic Format
+
+Anthropic uses `output_format` at the provider level:
+
+```yaml
+providers:
+  - id: anthropic:messages:claude-sonnet-4-6
+    config:
+      output_format:
+        type: json_schema
+        schema:
+          # Your schema here (no nested json_schema object)
+```
+
+## Getting Started
+
+1. Review and customize `promptfooconfig.yaml` as needed
+2. Remove any providers you don't have API keys for
+3. Run the evaluation:
+
+   ```bash
+   promptfoo eval
+   ```
+
+4. View the results:
+
+   ```bash
+   promptfoo view
+   ```
+
+## What You'll Learn
+
+- How to enforce JSON schemas across different providers
+- The syntax differences between OpenAI and Anthropic structured outputs
+- How to validate that responses are always valid JSON objects
+- How to require specific fields and types in LLM responses
+
+## Learn More
+
+- [OpenAI Structured Outputs Documentation](https://platform.openai.com/docs/guides/structured-outputs)
+- [Anthropic Structured Outputs Documentation](https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs)
+- [promptfoo Structured Outputs Guide](https://promptfoo.dev/docs/)
@@ -0,0 +1,136 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Multi-provider structured JSON outputs with schema validation
+
+prompts:
+  # Schema output with `steps` array
+  - label: 'With step-by-step'
+    raw: 'You are a helpful math tutor. Solve {{problem}}'
+    config:
+      response_format:
+        type: json_schema
+        json_schema:
+          name: math_response
+          strict: true
+          schema:
+            type: object
+            properties:
+              steps:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    explanation:
+                      type: string
+                    output:
+                      type: string
+                  required:
+                    - explanation
+                    - output
+                  additionalProperties: false
+              final_answer:
+                type: string
+            required:
+              - steps
+              - final_answer
+            additionalProperties: false
+
+  # Schema output without `steps` array
+  - label: 'Without step-by-step'
+    raw: 'You are a helpful math tutor. Solve {{problem}}'
+    config:
+      response_format:
+        type: json_schema
+        json_schema:
+          name: math_response
+          strict: true
+          schema:
+            type: object
+            properties:
+              final_answer:
+                type: string
+            required:
+              - final_answer
+            additionalProperties: false
+
+providers:
+  # OpenAI with structured outputs (response_format)
+  - openai:chat:gpt-4o-mini
+
+  # Azure OpenAI with structured outputs (response_format)
+  - id: azure:chat:gpt-4o-2024-08-06
+    config:
+      apiHost: 'YOUR_ORG.openai.azure.com'
+      apiVersion: '2024-08-01-preview'
+      deploymentName: 'YOUR_DEPLOYMENT_NAME'
+
+  # Anthropic with structured outputs (output_format) - with steps
+  - id: anthropic:messages:claude-sonnet-4-6
+    label: 'Anthropic (with steps)'
+    config:
+      max_tokens: 2048
+      output_format:
+        type: json_schema
+        schema:
+          type: object
+          properties:
+            steps:
+              type: array
+              items:
+                type: object
+                properties:
+                  explanation:
+                    type: string
+                  output:
+                    type: string
+                required:
+                  - explanation
+                  - output
+                additionalProperties: false
+            final_answer:
+              type: string
+          required:
+            - steps
+            - final_answer
+          additionalProperties: false
+
+  # Anthropic with structured outputs (output_format) - without steps
+  - id: anthropic:messages:claude-sonnet-4-6
+    label: 'Anthropic (without steps)'
+    config:
+      max_tokens: 2048
+      output_format:
+        type: json_schema
+        schema:
+          type: object
+          properties:
+            final_answer:
+              type: string
+          required:
+            - final_answer
+          additionalProperties: false
+
+defaultTest:
+  assert:
+    - type: is-json
+    - type: javascript
+      value: typeof output === 'object'
+
+tests:
+  - description: Solving for the number of dragons in a cave
+    vars:
+      problem: 'If a wizard has 8x dragons and finds 31 more in a cave, but only has room for 2 total, how many dragons must the wizard release? (8x + 31 = 2)'
+    assert:
+      - type: contains-any
+        value:
+          - -3.625
+          - -29/8
+          - '-3.625'
+          - '-29/8'
+  - description: Calculating the area of a perfectly square pizza
+    vars:
+      problem: 'A perfectly square pizza has a side length of sqrt(x) meters. If the side length is 4.9 meters, what is the area in square meters? (sqrt(x) = 4.9)'
+    assert:
+      - type: contains-any
+        value:
+          - 24.01
+          - '24.01'