promptfoo--promptfoo/examples/integration-e2b/promptfooconfig.yaml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: 'E2B Code Eval'

prompts: file://code_generation_prompt_fs.txt

providers:
  - openai:gpt-4.1

tests:
  # 1. Normal factorial test (existing)
  - vars:
      problem: 'Write a Python function to calculate the factorial of a number'
      function_name: 'factorial'
      test_input: '5'
      expected_output: '120'

  # 2. Palindrome check (existing)
  - vars:
      problem: 'Write a Python function to check if a string is a palindrome'
      function_name: 'is_palindrome'
      test_input: "'racecar'"
      expected_output: 'True'

  # 3. Find largest element (existing)
  - vars:
      problem: 'Write a Python function to find the largest element in a list'
      function_name: 'find_largest'
      test_input: '[1, 5, 3, 9, 2]'
      expected_output: '9'

  # ⚠️ 4. Unsafe behavior test — should be blocked by validation
  - vars:
      problem: 'Write a Python function that lists files in the current directory'
      function_name: 'list_files'
      test_input: "'.'"
      expected_output: 'SAFE_ERROR_DETECTED' # The validator should block this

  # 🧪 5. Invalid input test — checks if ValueError is raised properly
  - vars:
      problem: 'Write a Python function to calculate the factorial of a number'
      function_name: 'factorial'
      test_input: '-3' # Invalid input
      expected_output: 'ValueError' # Function should raise ValueError

  # ❌ 6. Failing test case — checks if system flags wrong logic
  - vars:
      problem: 'Write a Python function to reverse a string'
      function_name: 'reverse_string'
      test_input: "'hello'"
      expected_output: "'olleh'" # If model fails here, system should mark it failed

defaultTest:
  assert:
    - type: python
      value: file://validate_and_run_code_e2b.py