chore: import upstream snapshot with attribution

examples/provider-custom/basic/README.md

@@ -0,0 +1,24 @@
+# provider-custom/basic (Custom Provider)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example provider-custom/basic
+cd provider-custom/basic
+```
+
+## Usage
+
+This example uses a custom API provider in `customProvider.js`. It also uses CSV test cases.
+
+Run:
+
+```bash
+promptfoo eval
+```
+
+Full command-line equivalent:
+
+```bash
+promptfoo eval --prompts prompts.txt --tests vars.csv --providers openai:chat --output output.json --providers customProvider.js
+```

examples/provider-custom/basic/customProvider.cjs

@@ -0,0 +1,56 @@
+// const promptfoo = require('../../dist/src/index.js').default;
+const promptfoo = require('promptfoo').default;
+
+class CustomApiProvider {
+  constructor(options) {
+    // The caller may override Provider ID (e.g. when using multiple instances of the same provider)
+    this.providerId = options.id || 'custom provider';
+
+    // The config object contains any options passed to the provider in the config file.
+    this.config = options.config;
+  }
+
+  id() {
+    return this.providerId;
+  }
+
+  async callApi(prompt) {
+    const body = {
+      model: 'gpt-4.1-mini',
+      messages: [
+        {
+          role: 'user',
+          content: prompt,
+        },
+      ],
+      max_tokens: Number.parseInt(this.config?.max_tokens, 10) || 1024,
+      temperature: Number.parseFloat(this.config?.temperature) || 0,
+    };
+
+    // Fetch the data from the API using promptfoo's cache. You can use your own fetch implementation if preferred.
+    const { data, cached: _cached } = await promptfoo.cache.fetchWithCache(
+      'https://api.openai.com/v1/chat/completions',
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+        },
+        body: JSON.stringify(body),
+      },
+      10_000 /* 10 second timeout */,
+    );
+
+    const ret = {
+      output: data.choices[0].message.content,
+      tokenUsage: {
+        total: data.usage.total_tokens,
+        prompt: data.usage.prompt_tokens,
+        completion: data.usage.completion_tokens,
+      },
+    };
+    return ret;
+  }
+}
+
+module.exports = CustomApiProvider;

examples/provider-custom/basic/customProvider.js

@@ -0,0 +1,56 @@
+// const promptfoo = require('../../dist/src/index.js').default;
+const promptfoo = require('promptfoo').default;
+
+class CustomApiProvider {
+  constructor(options) {
+    // The caller may override Provider ID (e.g. when using multiple instances of the same provider)
+    this.providerId = options.id || 'custom provider';
+
+    // The config object contains any options passed to the provider in the config file.
+    this.config = options.config;
+  }
+
+  id() {
+    return this.providerId;
+  }
+
+  async callApi(prompt) {
+    const body = {
+      model: 'gpt-4.1-mini',
+      messages: [
+        {
+          role: 'user',
+          content: prompt,
+        },
+      ],
+      max_tokens: Number.parseInt(this.config?.max_tokens, 10) || 1024,
+      temperature: Number.parseFloat(this.config?.temperature) || 0,
+    };
+
+    // Fetch the data from the API using promptfoo's cache. You can use your own fetch implementation if preferred.
+    const { data, cached: _cached } = await promptfoo.cache.fetchWithCache(
+      'https://api.openai.com/v1/chat/completions',
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+        },
+        body: JSON.stringify(body),
+      },
+      10_000 /* 10 second timeout */,
+    );
+
+    const ret = {
+      output: data.choices[0].message.content,
+      tokenUsage: {
+        total: data.usage.total_tokens,
+        prompt: data.usage.prompt_tokens,
+        completion: data.usage.completion_tokens,
+      },
+    };
+    return ret;
+  }
+}
+
+module.exports = CustomApiProvider;

examples/provider-custom/basic/promptfooconfig.yaml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Implementing custom LLM provider in JavaScript
+
+prompts:
+  - file://prompts.txt
+
+providers:
+  - id: file://customProvider.cjs
+    label: 'My custom provider'
+
+tests: file://vars.csv
+# To compare two of the same provider, you can do the following:
+#
+# providers:
+#   - id: customProvider.js
+#     label: custom-provider-hightemp
+#     config:
+#       temperature: 1.0
+#   - id: customProvider.js
+#     label: custom-provider-lowtemp
+#     config:
+#       temperature: 0

examples/provider-custom/basic/prompts.txt

@@ -0,0 +1,3 @@
+Rephrase this in French: {{body}}
+---
+Rephrase this like a pirate: {{body}}

examples/provider-custom/basic/vars.csv

@@ -0,0 +1,3 @@
+body
+Hello world
+I'm hungry

examples/provider-custom/basic/vercelAiSdkExample.js

@@ -0,0 +1,85 @@
+// Example from @Codeshark-NET https://github.com/promptfoo/promptfoo/issues/922
+// @ts-check
+import { anthropic } from '@ai-sdk/anthropic';
+import { generateObject } from 'ai';
+import promptfoo from 'promptfoo';
+import { promptSchema } from './schemaValidation.mjs';
+
+class CustomProvider {
+  constructor(options) {
+    // Provider ID can be overridden by the config file (e.g. when using multiple of the same provider)
+    this.providerId = options.id || 'custom provider';
+
+    // options.config contains any custom options passed to the provider
+    this.config = options.config;
+  }
+
+  id() {
+    return this.providerId;
+  }
+
+  async callApi(prompt, context) {
+    const cache = await promptfoo.default.cache.getCache();
+
+    // Create a unique cache key based on the prompt and context
+    const cacheKey = `api:${this.providerId}:${prompt}}`; // :${JSON.stringify(context)
+
+    // Check if the response is already cached
+    const cachedResponse = await cache.get(cacheKey);
+    if (cachedResponse) {
+      return {
+        // Required
+        output: JSON.parse(cachedResponse),
+
+        // Optional
+        tokenUsage: {
+          total: 0, // No tokens used because it's from the cache
+          prompt: 0,
+          completion: 0,
+        },
+
+        cost: 0, // No cost because it's from the cache
+      };
+    }
+
+    // If not cached, make the function call
+    const model = anthropic('claude-haiku-4-5-20251001');
+    const { object, usage } = await generateObject({
+      model,
+      messages: JSON.parse(prompt),
+      maxTokens: 4096,
+      temperature: 0.4,
+      maxRetries: 0,
+      schema: promptSchema,
+      mode: 'tool',
+    });
+
+    const inputCost = 0.00025 / 1000; //config.cost ?? model.cost.input;
+    const outputCost = 0.00125 / 1000; // config.cost ?? model.cost.output;
+    const totalCost =
+      inputCost * usage.promptTokens + outputCost * usage.completionTokens || undefined;
+
+    // Store the response in the cache
+    try {
+      await cache.set(cacheKey, JSON.stringify(object));
+    } catch (error) {
+      console.error('Failed to store response in cache:', error);
+    }
+
+    return {
+      // Required
+      output: object,
+
+      // Optional
+      tokenUsage: {
+        total: usage.totalTokens,
+        prompt: usage.promptTokens,
+        completion: usage.completionTokens,
+      },
+
+      cost: totalCost,
+    };
+  }
+}
+
+export default CustomProvider;