chore: import upstream snapshot with attribution

examples/eval-errors-vs-failures/README.md

@@ -0,0 +1,12 @@
+# eval-errors-vs-failures (Errors Vs Failures)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example eval-errors-vs-failures
+cd eval-errors-vs-failures
+```
+
+## Usage
+
+This directory contains a custom provider that throws errors when the prompt includes `!ERROR!`, demonstrating how promptfoo distinguishes between test errors (provider failures) and test failures (failed assertions).

examples/eval-errors-vs-failures/customProvider.js

@@ -0,0 +1,53 @@
+class CustomApiProvider {
+  constructor(options) {
+    // The caller may override Provider ID (e.g. when using multiple instances of the same provider)
+    this.providerId = options.id || 'custom provider';
+
+    // The config object contains any options passed to the provider in the config file.
+    this.config = options.config;
+  }
+
+  id() {
+    return this.providerId;
+  }
+
+  async callApi(prompt) {
+    if (prompt.includes('!ERROR!')) {
+      throw new Error('Forced error: ' + prompt);
+    }
+    const body = {
+      model: 'gpt-4.1-mini',
+      messages: [
+        {
+          role: 'user',
+          content: prompt,
+        },
+      ],
+      max_tokens: Number.parseInt(this.config?.max_tokens, 10) || 1024,
+      temperature: Number.parseFloat(this.config?.temperature) || 0,
+    };
+
+    const response = await fetch('https://api.openai.com/v1/chat/completions', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+      },
+      body: JSON.stringify(body),
+    });
+
+    const data = await response.json();
+
+    const ret = {
+      output: data.choices[0].message.content,
+      tokenUsage: {
+        total: data.usage.total_tokens,
+        prompt: data.usage.prompt_tokens,
+        completion: data.usage.completion_tokens,
+      },
+    };
+    return ret;
+  }
+}
+
+module.exports = CustomApiProvider;

examples/eval-errors-vs-failures/promptfooconfig.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+
+description: 'Errors vs Failures'
+
+prompts:
+  - 'Tell me about {{topic}}'
+
+providers:
+  - id: file://customProvider.js
+    label: 'My custom provider'
+
+tests:
+  - vars:
+      topic: 'black holes'
+    assert:
+      # This should succeed
+      - type: not-equals
+        value: this should always pass
+  - vars:
+      # Force an error
+      topic: '!ERROR! banana slugs'
+  - vars:
+      topic: 'blue herons'
+    assert:
+      # Force a failure
+      - type: equals
+        value: this should always fail

examples/eval-errors-vs-failures/prompts.txt

@@ -0,0 +1,3 @@
+Rephrase this in French: {{body}}
+---
+Rephrase this like a pirate: {{body}}

examples/eval-errors-vs-failures/vars.csv

@@ -0,0 +1,3 @@
+body
+Hello world
+I'm hungry