chore: import upstream snapshot with attribution

examples/config-node-package/README.md

@@ -0,0 +1,47 @@
+# config-node-package (Node Package)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example config-node-package
+cd config-node-package
+```
+
+This example demonstrates using promptfoo from a Node.js script.
+
+## Prerequisites
+
+- Node.js ^20.20.0 or >=22.22.0 (Node.js 20 support ends July 30, 2026; Node.js 24 LTS recommended)
+- API keys for LLM providers set as environment variables:
+  - `OPENAI_API_KEY` - Get from [OpenAI API keys page](https://platform.openai.com/api-keys)
+  - `ANTHROPIC_API_KEY` - Get from [Anthropic Console](https://console.anthropic.com/) (optional)
+
+You can set these in a `.env` file:
+
+```text
+OPENAI_API_KEY=sk-your-key-here
+ANTHROPIC_API_KEY=sk-ant-your-key-here
+```
+
+## Running the Example
+
+1. Install dependencies:
+
+```bash
+npm install
+```
+
+2. Execute the script:
+
+```bash
+node full-eval.js
+```
+
+## Expected Results
+
+The script will:
+
+- Run evaluations programmatically using the promptfoo Node.js API
+- Save results to `output.json`
+- Display evaluation metrics in the console
+- Allow you to view detailed results with `promptfoo view`

examples/config-node-package/full-eval.js

@@ -0,0 +1,134 @@
+import fs from 'node:fs';
+
+import promptfoo from 'promptfoo';
+
+const prompts = [
+  // Prompts can be raw text...
+  'Write a haiku about: {{body}}',
+
+  // Or message arrays...
+  [
+    {
+      role: 'system',
+      content: 'You are speaking to a time traveler. Begin your conversation.',
+    },
+    {
+      role: 'user',
+      content: '{{body}}',
+    },
+  ],
+
+  // Or files
+  'file://prompt.txt',
+
+  // Or functions
+  (vars) => {
+    const genres = ['fantasy', 'sci-fi', 'mystery', 'horror'];
+    const characters = {
+      fantasy: 'wizard',
+      'sci-fi': 'alien',
+      mystery: 'detective',
+      horror: 'ghost',
+    };
+    const genre = genres[Math.floor(Math.random() * genres.length)];
+    const character = characters[genre];
+    return [
+      {
+        role: 'system',
+        content: `You have encountered a ${character} in a ${genre} setting. Engage with the character using the knowledge typical for that genre.`,
+      },
+      {
+        role: 'user',
+        content: '{{body}}',
+      },
+    ];
+  },
+];
+
+const customFunction = (prompt, context) => {
+  // Call an LLM API in this function, or any other logic.
+  console.log(`Prompt: ${prompt}, vars: ${JSON.stringify(context.vars)}`);
+  return {
+    output: '<LLM output>',
+  };
+};
+customFunction.label = 'My custom function';
+
+const providers = [
+  // Call the OpenAI API GPT 4o Mini
+  'openai:gpt-4.1-mini',
+
+  // Or use a custom function.
+  customFunction,
+];
+
+const tests = [
+  {
+    vars: {
+      body: 'Hello world',
+    },
+  },
+  {
+    vars: {
+      body: "I'm hungry",
+    },
+    assert: [
+      {
+        type: 'javascript',
+        value: (output) => {
+          // Logic for checking LLM output goes here...
+          const pass = output.includes('foo bar');
+          return {
+            pass,
+            score: pass ? 1.0 : 0.0,
+            reason: pass ? 'Output contained substring' : 'Output did not contain substring',
+          };
+        },
+      },
+    ],
+  },
+  {
+    vars: {
+      body: 'Tell me a joke',
+    },
+    options: {
+      // Transform functions can be passed directly when using the node package.
+      transform: (output) => output.toLowerCase(),
+    },
+    assert: [
+      {
+        type: 'contains',
+        value: 'joke',
+        // Per-assertion transforms also accept functions.
+        transform: (output) => output.trim(),
+      },
+    ],
+  },
+];
+
+(async () => {
+  const evalRecord = await promptfoo.evaluate({
+    prompts,
+    providers,
+    tests,
+
+    // Persist results locally so they are visible in the promptfoo viewer
+    writeLatestResults: true,
+
+    // Uncomment to enable sharing (requires cloud account or self-hosted server)
+    // sharing: true,
+  });
+  const results = await evalRecord.toEvaluateSummary();
+
+  console.log('RESULTS:');
+  const resultsString = JSON.stringify(results, null, 2);
+  console.log(resultsString);
+
+  // Display shareable URL if available
+  if (results.shareableUrl) {
+    console.log('\nView results online:', results.shareableUrl);
+  }
+
+  fs.writeFileSync('output.json', resultsString);
+  console.log('Wrote output.json');
+})();

examples/config-node-package/output.json

@@ -0,0 +1,385 @@
+{
+  "_resultsLoaded": false,
+  "createdAt": 1758065695513,
+  "id": "eval-IF3-2025-09-16T23:34:55",
+  "config": {
+    "prompts": [
+      {
+        "raw": "Write a haiku about: {{body}}",
+        "label": "Write a haiku about: {{body}}"
+      },
+      {
+        "raw": "[{\"role\":\"system\",\"content\":\"You are speaking to a time traveler. Begin your conversation.\"},{\"role\":\"user\",\"content\":\"{{body}}\"}]",
+        "label": "[{\"role\":\"system\",\"content\":\"You are speaking to a time traveler. Begin your conversation.\"},{\"role\":\"user\",\"content\":\"{{body}}\"}]"
+      },
+      {
+        "raw": "Write 3 words about {{body}}",
+        "label": "prompt.txt: Write 3 words about {{body}}"
+      },
+      {
+        "raw": "(vars) => {\n    const genres = ['fantasy', 'sci-fi', 'mystery', 'horror'];\n    const characters = {\n      fantasy: 'wizard',\n      'sci-fi': 'alien',\n      mystery: 'detective',\n      horror: 'ghost',\n    };\n    const genre = genres[Math.floor(Math.random() * genres.length)];\n    const character = characters[genre];\n    return [\n      {\n        role: 'system',\n        content: `You have encountered a ${character} in a ${genre} setting. Engage with the character using the knowledge typical for that genre.`,\n      },\n      {\n        role: 'user',\n        content: '{{body}}',\n      },\n    ];\n  }",
+        "label": ""
+      }
+    ],
+    "providers": ["openai:gpt-4.1-mini", null],
+    "tests": [
+      {
+        "vars": {
+          "body": "Hello world"
+        }
+      },
+      {
+        "vars": {
+          "body": "I'm hungry"
+        },
+        "assert": [
+          {
+            "type": "javascript"
+          }
+        ]
+      }
+    ],
+    "writeLatestResults": true
+  },
+  "results": [],
+  "prompts": [
+    {
+      "raw": "Write a haiku about: {{body}}",
+      "label": "Write a haiku about: {{body}}",
+      "id": "64a38a678decacb9c6c12748f049b7558eaf709be7fe34f2b7120605754a02e6",
+      "provider": "openai:gpt-4.1-mini",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 40,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 68,
+          "total": 68,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0.00007280000000000001
+      }
+    },
+    {
+      "raw": "[{\"role\":\"system\",\"content\":\"You are speaking to a time traveler. Begin your conversation.\"},{\"role\":\"user\",\"content\":\"{{body}}\"}]",
+      "label": "[{\"role\":\"system\",\"content\":\"You are speaking to a time traveler. Begin your conversation.\"},{\"role\":\"user\",\"content\":\"{{body}}\"}]",
+      "id": "13fa8ee48f9101367ab54fddbb72cc53eabc1aeacc250503e0023c99635059c7",
+      "provider": "openai:gpt-4.1-mini",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 74,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 91,
+          "total": 91,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0.00008560000000000001
+      }
+    },
+    {
+      "raw": "Write 3 words about {{body}}",
+      "label": "prompt.txt: Write 3 words about {{body}}",
+      "id": "16946aa0e5135cee46b889df493c8c8c6033b45b679c47143e94e39ee1854e26",
+      "provider": "openai:gpt-4.1-mini",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 55,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 37,
+          "total": 37,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0.000025600000000000002
+      }
+    },
+    {
+      "raw": "(vars) => {\n    const genres = ['fantasy', 'sci-fi', 'mystery', 'horror'];\n    const characters = {\n      fantasy: 'wizard',\n      'sci-fi': 'alien',\n      mystery: 'detective',\n      horror: 'ghost',\n    };\n    const genre = genres[Math.floor(Math.random() * genres.length)];\n    const character = characters[genre];\n    return [\n      {\n        role: 'system',\n        content: `You have encountered a ${character} in a ${genre} setting. Engage with the character using the knowledge typical for that genre.`,\n      },\n      {\n        role: 'user',\n        content: '{{body}}',\n      },\n    ];\n  }",
+      "label": "",
+      "id": "cffda0450e7d3fc46ddfe961b52b57210717fab072e65d6814015e756ad56b4f",
+      "provider": "openai:gpt-4.1-mini",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 2207,
+        "tokenUsage": {
+          "prompt": 35,
+          "completion": 22,
+          "cached": 116,
+          "total": 173,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0.00019280000000000002
+      }
+    },
+    {
+      "raw": "Write a haiku about: {{body}}",
+      "label": "Write a haiku about: {{body}}",
+      "id": "64a38a678decacb9c6c12748f049b7558eaf709be7fe34f2b7120605754a02e6",
+      "provider": "My custom function",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 0,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 0,
+          "total": 0,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0
+      }
+    },
+    {
+      "raw": "[{\"role\":\"system\",\"content\":\"You are speaking to a time traveler. Begin your conversation.\"},{\"role\":\"user\",\"content\":\"{{body}}\"}]",
+      "label": "[{\"role\":\"system\",\"content\":\"You are speaking to a time traveler. Begin your conversation.\"},{\"role\":\"user\",\"content\":\"{{body}}\"}]",
+      "id": "13fa8ee48f9101367ab54fddbb72cc53eabc1aeacc250503e0023c99635059c7",
+      "provider": "My custom function",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 0,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 0,
+          "total": 0,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0
+      }
+    },
+    {
+      "raw": "Write 3 words about {{body}}",
+      "label": "prompt.txt: Write 3 words about {{body}}",
+      "id": "16946aa0e5135cee46b889df493c8c8c6033b45b679c47143e94e39ee1854e26",
+      "provider": "My custom function",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 1,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 0,
+          "total": 0,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0
+      }
+    },
+    {
+      "raw": "(vars) => {\n    const genres = ['fantasy', 'sci-fi', 'mystery', 'horror'];\n    const characters = {\n      fantasy: 'wizard',\n      'sci-fi': 'alien',\n      mystery: 'detective',\n      horror: 'ghost',\n    };\n    const genre = genres[Math.floor(Math.random() * genres.length)];\n    const character = characters[genre];\n    return [\n      {\n        role: 'system',\n        content: `You have encountered a ${character} in a ${genre} setting. Engage with the character using the knowledge typical for that genre.`,\n      },\n      {\n        role: 'user',\n        content: '{{body}}',\n      },\n    ];\n  }",
+      "label": "",
+      "id": "cffda0450e7d3fc46ddfe961b52b57210717fab072e65d6814015e756ad56b4f",
+      "provider": "My custom function",
+      "metrics": {
+        "score": 1,
+        "testPassCount": 1,
+        "testFailCount": 1,
+        "testErrorCount": 0,
+        "assertPassCount": 0,
+        "assertFailCount": 1,
+        "totalLatencyMs": 0,
+        "tokenUsage": {
+          "prompt": 0,
+          "completion": 0,
+          "cached": 0,
+          "total": 0,
+          "numRequests": 2,
+          "completionDetails": {
+            "reasoning": 0,
+            "acceptedPrediction": 0,
+            "rejectedPrediction": 0
+          },
+          "assertions": {
+            "total": 0,
+            "prompt": 0,
+            "completion": 0,
+            "cached": 0,
+            "numRequests": 0,
+            "completionDetails": {
+              "reasoning": 0,
+              "acceptedPrediction": 0,
+              "rejectedPrediction": 0
+            }
+          }
+        },
+        "namedScores": {},
+        "namedScoresCount": {},
+        "cost": 0
+      }
+    }
+  ],
+  "persisted": true,
+  "vars": ["body"]
+}

examples/config-node-package/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "simple-import",
+  "version": "1.0.0",
+  "license": "MIT",
+  "private": true,
+  "type": "module",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "promptfoo": "latest"
+  }
+}

examples/config-node-package/prompt.txt

@@ -0,0 +1 @@
+Write 3 words about {{body}}