promptfoo--promptfoo/examples/eval-json-output/promptfooconfig.yaml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: 'JSON object evals'

prompts:
  - 'Output a JSON object that contains the keys `color` and `countries`, describing the following object: {{item}}'

providers:
  - id: openai:chat:gpt-4.1-mini
    config:
      response_format:
        type: json_object

tests:
  - vars:
      item: Banana
    assert:
      - type: is-json
      - type: javascript
        # Parse the JSON and test the contents
        value: JSON.parse(output).color === 'yellow' && JSON.parse(output).countries.includes('Ecuador')

  - vars:
      item: Passion fruit
    options:
      # Parse the JSON before all assertions
      transform: JSON.parse(output)
    assert:
      - type: is-json
        # Optional JSON schema
        value:
          required:
            - 'color'
            - 'countries'
          type: object
          properties:
            color:
              type: string
            countries:
              type: array
              items:
                type: string
      - type: javascript
        # JSON.parse is no longer necessary here
        value: output.color === 'purple' && output.countries.includes('Brazil')
      - type: contains-any
        transform: output.countries
        value:
          - Guatemala
          - Costa Rica
          - India
          - Indonesia
# Uncomment to parse JSON outputs for ALL tests
# defaultTest:
#   options:
#     transform: JSON.parse(output)