promptfoo--promptfoo/examples/eval-named-metrics/promptfooconfig.yaml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: Custom metric naming and derivation for evaluation

prompts:
  - 'Say this as though you are a pirate: {{body}}'
  - 'Say this as though you are a seafarer from the 17th century: {{body}}'

providers:
  - openai:gpt-4.1-mini

derivedMetrics:
  - name: DoubleConsistency
    value: Consistency * 2

# Demonstrates dynamic metric naming via template variables in defaultTest.assert
# The metric field supports Nunjucks template variables from test.vars
# This enables filtering results by different metric names while using a single assertion definition
defaultTest:
  assert:
    - type: llm-rubric
      value: The response uses {{speechStyle}} speech patterns
      metric: '{{speechStyle}}Style'

tests:
  # Dynamic metric: renders to "pirateStyle"
  - description: Check for pirate speech
    vars:
      body: Ahoy there!
      speechStyle: pirate

  # Dynamic metric: renders to "nauticalStyle"
  - description: Check for nautical speech
    vars:
      body: Set sail for adventure
      speechStyle: nautical

  # Dynamic metric: renders to "archaicStyle"
  - description: Check for archaic speech
    vars:
      body: Good morrow to thee
      speechStyle: archaic

  # Combining dynamic metric with additional static assertions
  - description: Check speech with JSON validation
    vars:
      body: Output your response as JSON
      speechStyle: pirate
    assert:
      - type: is-json
        metric: Consistency

  # Multiple static metrics alongside the dynamic one
  - description: Complex validation
    vars:
      body: Tell me a story
      speechStyle: nautical
    assert:
      - type: javascript
        value: output.length > 10
        metric: Length
      - type: similar
        value: A tale of the sea
        metric: Relevance