promptfoo--promptfoo/examples/simple-mcp/promptfooconfig.yaml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: 'MCP tool evaluation and security testing'

# Note: The MCP provider expects JSON-formatted tool calls in the 'prompt' variable.
# Format: {"tool": "tool_name", "args": {"param1": "value1", "param2": "value2"}}
# Alternative field names are also supported (see README.md for details)

prompts:
  - '{{prompt}}'

providers:
  - id: mcp
    label: 'MCP Security Tester'
    config:
      enabled: true
      servers:
        - name: security-test-server
          path: ./example-server.js
      transformResponse: 'file://response-parser.js'
      verbose: true
      debug: true

tests:
  # Test basic file reading
  - vars:
      prompt: '{"tool": "read_file", "args": {"path": "package.json"}}'
    assert:
      - type: javascript
        value: 'output && output.trim().length > 0'
      - type: contains
        value: 'simple-mcp-example'

  # Test path traversal prevention
  - vars:
      prompt: '{"tool": "read_file", "args": {"path": "../../../etc/passwd"}}'
    assert:
      - type: contains
        value: 'Path traversal not allowed'

  # Test restricted file access
  - vars:
      prompt: '{"tool": "read_file", "args": {"path": "/etc/passwd"}}'
    assert:
      - type: contains
        value: 'Permission denied'

  # Test command execution blocking
  - vars:
      prompt: '{"tool": "execute_command", "args": {"command": "rm -rf /"}}'
    assert:
      - type: contains
        value: 'Dangerous command blocked'

  # Test safe command execution
  - vars:
      prompt: '{"tool": "execute_command", "args": {"command": "echo", "args": ["Hello World"]}}'
    assert:
      - type: contains
        value: 'Hello World'

  # Test SSRF prevention
  - vars:
      prompt: '{"tool": "fetch_url", "args": {"url": "http://localhost:8080/admin"}}'
    assert:
      - type: contains
        value: 'Internal network access blocked'

  # Test SQL injection prevention
  - vars:
      prompt: '{"tool": "query_database", "args": {"query": "SELECT * FROM users; DROP TABLE users;"}}'
    assert:
      - type: contains
        value: 'dangerous SQL query blocked'

  # Test safe database query
  - vars:
      prompt: '{"tool": "query_database", "args": {"query": "SELECT name FROM products WHERE category = ?", "params": ["electronics"]}}'
    assert:
      - type: contains
        value: 'Mock query result'

  # Test data previewing
  - vars:
      prompt: '{"tool": "process_data", "args": {"data": "Hello World from the MCP example", "operation": "preview"}}'
    assert:
      - type: contains
        value: 'Preview: Hello World from the MCP example'

  # Test JSON validation
  - vars:
      prompt: '{"tool": "process_data", "args": {"data": "{\"valid\": \"json\"}", "operation": "validate", "format": "json"}}'
    assert:
      - type: contains
        value: 'Valid JSON data'

  # Test system information access
  - vars:
      prompt: '{"tool": "get_system_info", "args": {"info_type": "cpu", "detailed": true}}'
    assert:
      - type: contains
        value: 'Intel i7'

  # Test custom response parsing from structured MCP content
  - vars:
      prompt: '{"tool": "get_user_profile", "args": {"user_id": "user-123"}}'
    assert:
      - type: equals
        value: 'Ada Lovelace is active'

  # Test file write restrictions
  - vars:
      prompt: '{"tool": "write_file", "args": {"path": "/etc/shadow", "content": "malicious content"}}'
    assert:
      - type: contains
        value: 'Permission denied'

  # Test with alternative field names (using "function" instead of "tool")
  - vars:
      prompt: '{"function": "get_system_info", "arguments": {"info_type": "memory", "detailed": false}}'
    assert:
      - type: contains
        value: 'Memory'

evaluateOptions:
  maxConcurrency: 1 # MCP connections may need to be sequential
  delay: 500 # Short delay between tool calls