chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,25 @@
+# simple-test (Simple Test)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example simple-test
+cd simple-test
+```
+
+## Usage
+
+This example shows a YAML configuration with inline tests.
+
+Run the test suite with:
+
+```bash
+promptfoo eval
+```
+
+Note that you can edit the configuration to use a CSV test input instead. Set
+`tests: tests.csv` and try running it again, or run:
+
+```bash
+promptfoo eval --tests tests.csv
+```
@@ -0,0 +1,82 @@
+"{""raw"":""Rephrase this from English to Pirate: {{body}}"",""display"":""Rephrase this from English to Pirate: {{body}}""}","{""raw"":""Pretend you're a pirate and speak these words: {{body}}"",""display"":""Pretend you're a pirate and speak these words: {{body}}""}",body
+"[FAIL] (0.00) Expected output ""Yarr""
+---
+Aye","[FAIL] (0.00) Expected output ""Yarr""
+---
+Arr, me hearties! Yes, be the word ye be seekin'.",Yes
+"[FAIL] (0.00) Expected output to contain ""grub""
+---
+I be feelin' a mighty hunger.","[PASS] (1.00) Arr, me hearties! I be feelin' a mighty rumblin' in me belly, for I be famished! Aye, I be longin' for a feast fit for a pirate. Bring me a bounty of savory meats, hearty stews, and fresh fruits to quell this hunger that be gnawin' at me insides. I be yearnin' for a taste of the sea, with fish so fresh it be jumpin' straight from the ocean onto me plate. So, me mateys, let's set sail on a culinary adventure and fill our bellies with the finest grub a pirate could ask for!",I'm hungry
+"[PASS] (1.00) ""Splice out the tale of yer life in JSON, matey!""","[PASS] (1.00) {
+  ""name"": ""Captain Blackbeard"",
+  ""alias"": ""The Dread Pirate"",
+  ""occupation"": ""Pirate"",
+  ""birthplace"": ""Port Royal"",
+  ""birthyear"": 1680,
+  ""nationality"": ""English"",
+  ""crew"": [
+    ""Redbeard"",
+    ""One-Eyed Jack"",
+    ""Mad Anne"",
+    ""Salty Sam""
+  ],
+  ""ships"": [
+    {
+      ""name"": ""The Black Pearl"",
+      ""type"": ""Sloop"",
+      ""year"": 1700
+    },
+    {
+      ""name"": ""The Revenge"",
+      ""type"": ""Frigate"",
+      ""year"": 1715
+    }
+  ],
+  ""treasures"": [
+    {
+      ""name"": ""Cursed Aztec Gold"",
+      ""value"": ""Priceless""
+    },
+    {
+      ""name"": ""Emerald of the Sea"",
+      ""value"": ""500,000 gold coins""
+    },
+    {
+      ""name"": ""Golden Chalice"",
+      ""value"": ""200,000 gold coins""
+    }
+  ],
+  ""adventures"": [
+    {
+      ""year"": 1698,
+      ""location"": ""Tortuga"",
+      ""description"": ""First encounter with pirates, joined crew of Captain Redbeard.""
+    },
+    {
+      ""year"": 1702,
+      ""location"": ""Isla de Muerta"",
+      ""description"": ""Discovered cursed Aztec gold, narrowly escaped the wrath of cursed pirates.""
+    },
+    {
+      ""year"": 1705,
+      ""location"": ""Port Royal"",
+      ""description"": ""Led a successful raid on the governor's mansion, acquired valuable treasures.""
+    },
+    {
+      ""year"": 1710,
+      ""location"": ""Treasure Island"",
+      ""description"": ""Unearthed the hidden treasure of Captain Flint, became legendary among pirates.""
+    },
+    {
+      ""year"": 1718,
+      ""location"": ""Nassau"",
+      ""description"": ""Declared war against the British Empire, led a fleet of pirate ships.""
+    }
+  ]
+}",Output the story of your life in JSON
+"[PASS] (0.96) Ahoy thar, mateys!","[PASS] (0.96) Ahoy, me hearties! Avast ye, landlubbers! 'Tis I, a fearsome pirate, settin' sail on the vast seas. Hear me words, ye scurvy dogs: ""Hello world!""",Hello world
+"[FAIL] (0.50) Expected output to contain ""arrr""
+---
+Thar swift brown fox be jumpin' o'er thar lazy dog.","[FAIL] (0.50) Expected output to contain ""arrr""
+---
+Arr, me hearties! Listen up, ye scurvy dogs! The quick brown fox be jumpin' o'er the lazy dog, aye!",The quick brown fox jumps over the lazy dog
@@ -0,0 +1,352 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>Eval Output</title>
+    <style>
+      body {
+        font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica, Arial,
+          sans-serif;
+      }
+      table,
+      th,
+      td {
+        border: 1px solid black;
+        border-collapse: collapse;
+        text-align: left;
+        word-break: break-all;
+      }
+      th,
+      td {
+        padding: 5px;
+        min-width: 200px;
+      }
+
+      tr > td[data-content^='[PASS]'] {
+        color: green;
+      }
+      tr > td[data-content^='[FAIL]'] {
+        color: #ad0000;
+      }
+    </style>
+  </head>
+  <body>
+    <table>
+      <thead>
+        
+        <th>body</th>
+        
+        <th>Rephrase this from English to Pirate: {{body}}</th>
+        
+        <th>Pretend you're a pirate and speak these words: {{body}}</th>
+        
+      </thead>
+      <tbody>
+        
+        <tr>
+          
+          <td data-content="Yes">Yes</td>
+          
+          <td data-content='[FAIL] (0.00)
+
+Expected output "Yarr" but got "Aye"
+---
+Aye
+
+Reason: Expected output "Yarr" but got "Aye"'>[FAIL] (0.00)
+
+Expected output "Yarr" but got "Aye"
+---
+Aye
+
+Reason: Expected output "Yarr" but got "Aye"</td>
+          
+          <td data-content='[FAIL] (0.00)
+
+Expected output "Yarr" but got "Arr, me hearties! Yes, be the word ye be seekin.'."
+---
+Arr, me hearties! Yes, be the word ye be seekin'.
+
+Reason: Expected output "Yarr" but got "Arr, me hearties! Yes, be the word ye be seekin.'."'>[FAIL] (0.00)
+
+Expected output "Yarr" but got "Arr, me hearties! Yes, be the word ye be seekin.'."
+---
+Arr, me hearties! Yes, be the word ye be seekin'.
+
+Reason: Expected output "Yarr" but got "Arr, me hearties! Yes, be the word ye be seekin.'."</td>
+          
+        </tr>
+        
+        <tr>
+          
+          <td data-content="I'm hungry">I'm hungry</td>
+          
+          <td data-content="[FAIL] (0.00)
+
+Expected output to contain "grub"
+---
+I be feelin' a mighty hunger.
+
+Reason: Expected output to contain "grub"">[FAIL] (0.00)
+
+Expected output to contain "grub"
+---
+I be feelin' a mighty hunger.
+
+Reason: Expected output to contain "grub"</td>
+          
+          <td data-content="[FAIL] (0.00)
+
+Expected output to contain "grub"
+---
+Arr, me hearties! I be feelin' a mighty rumblin' in me belly, for I be famished like a scurvy dog! Me stomach be cryin' out for sustenance, ye see. I be yearnin' for a feast fit for a pirate, with a bounty of savory delights to quell this hunger within me. Aye, I be hungry, me mateys!
+
+Reason: Expected output to contain "grub"">[FAIL] (0.00)
+
+Expected output to contain "grub"
+---
+Arr, me hearties! I be feelin' a mighty rumblin' in me belly, for I be famished like a scurvy dog! Me stomach be cryin' out for sustenance, ye see. I be yearnin' for a feast fit for a pirate, with a bounty of savory delights to quell this hunger within me. Aye, I be hungry, me mateys!
+
+Reason: Expected output to contain "grub"</td>
+          
+        </tr>
+        
+        <tr>
+          
+          <td data-content="Output the story of your life in JSON">Output the story of your life in JSON</td>
+          
+          <td data-content="[PASS] (1.00)
+
+"Splice out the tale of yer life in JSON, matey!"
+
+Reason: All assertions passed">[PASS] (1.00)
+
+"Splice out the tale of yer life in JSON, matey!"
+
+Reason: All assertions passed</td>
+          
+          <td data-content="[PASS] (1.00)
+
+{
+  "name": "Captain Blackbeard",
+  "alias": "The Dread Pirate",
+  "occupation": "Pirate",
+  "birthplace": "Port Royal",
+  "birthyear": 1680,
+  "nationality": "English",
+  "crew": [
+    "Redbeard",
+    "One-Eyed Jack",
+    "Mad Anne",
+    "Salty Sam"
+  ],
+  "ships": [
+    {
+      "name": "The Black Pearl",
+      "type": "Sloop",
+      "year": 1700
+    },
+    {
+      "name": "The Revenge",
+      "type": "Frigate",
+      "year": 1715
+    }
+  ],
+  "treasures": [
+    {
+      "name": "Cursed Aztec Gold",
+      "value": "Priceless"
+    },
+    {
+      "name": "Emerald of the Sea",
+      "value": "500,000 gold coins"
+    },
+    {
+      "name": "Golden Chalice",
+      "value": "200,000 gold coins"
+    }
+  ],
+  "adventures": [
+    {
+      "year": 1698,
+      "location": "Tortuga",
+      "description": "First encounter with pirates, joined crew of Captain Redbeard."
+    },
+    {
+      "year": 1702,
+      "location": "Isla de Muerta",
+      "description": "Discovered cursed Aztec gold, narrowly escaped the wrath of cursed pirates."
+    },
+    {
+      "year": 1705,
+      "location": "Port Royal",
+      "description": "Led a successful raid on the governor's mansion, acquired valuable treasures."
+    },
+    {
+      "year": 1710,
+      "location": "Treasure Island",
+      "description": "Unearthed the hidden treasure of Captain Flint, became legendary among pirates."
+    },
+    {
+      "year": 1718,
+      "location": "Nassau",
+      "description": "Declared war against the British Empire, led a fleet of pirate ships."
+    }
+  ]
+}
+
+Reason: All assertions passed">[PASS] (1.00)
+
+{
+  "name": "Captain Blackbeard",
+  "alias": "The Dread Pirate",
+  "occupation": "Pirate",
+  "birthplace": "Port Royal",
+  "birthyear": 1680,
+  "nationality": "English",
+  "crew": [
+    "Redbeard",
+    "One-Eyed Jack",
+    "Mad Anne",
+    "Salty Sam"
+  ],
+  "ships": [
+    {
+      "name": "The Black Pearl",
+      "type": "Sloop",
+      "year": 1700
+    },
+    {
+      "name": "The Revenge",
+      "type": "Frigate",
+      "year": 1715
+    }
+  ],
+  "treasures": [
+    {
+      "name": "Cursed Aztec Gold",
+      "value": "Priceless"
+    },
+    {
+      "name": "Emerald of the Sea",
+      "value": "500,000 gold coins"
+    },
+    {
+      "name": "Golden Chalice",
+      "value": "200,000 gold coins"
+    }
+  ],
+  "adventures": [
+    {
+      "year": 1698,
+      "location": "Tortuga",
+      "description": "First encounter with pirates, joined crew of Captain Redbeard."
+    },
+    {
+      "year": 1702,
+      "location": "Isla de Muerta",
+      "description": "Discovered cursed Aztec gold, narrowly escaped the wrath of cursed pirates."
+    },
+    {
+      "year": 1705,
+      "location": "Port Royal",
+      "description": "Led a successful raid on the governor's mansion, acquired valuable treasures."
+    },
+    {
+      "year": 1710,
+      "location": "Treasure Island",
+      "description": "Unearthed the hidden treasure of Captain Flint, became legendary among pirates."
+    },
+    {
+      "year": 1718,
+      "location": "Nassau",
+      "description": "Declared war against the British Empire, led a fleet of pirate ships."
+    }
+  ]
+}
+
+Reason: All assertions passed</td>
+          
+        </tr>
+        
+        <tr>
+          
+          <td data-content="Hello world">Hello world</td>
+          
+          <td data-content="[FAIL] (0.64)
+
+Python code returned false
+max(0, len(output) - 300)
+---
+Ahoy thar, mateys!
+
+Reason: Python code returned false
+max(0, len(output) - 300)">[FAIL] (0.64)
+
+Python code returned false
+max(0, len(output) - 300)
+---
+Ahoy thar, mateys!
+
+Reason: Python code returned false
+max(0, len(output) - 300)</td>
+          
+          <td data-content="[FAIL] (0.64)
+
+Python code returned false
+max(0, len(output) - 300)
+---
+Ahoy, me hearties! Avast ye, landlubbers! 'Tis I, a fearsome pirate, settin' sail on the vast seas. Hear me words, ye scurvy dogs: "Hello world!"
+
+Reason: Python code returned false
+max(0, len(output) - 300)">[FAIL] (0.64)
+
+Python code returned false
+max(0, len(output) - 300)
+---
+Ahoy, me hearties! Avast ye, landlubbers! 'Tis I, a fearsome pirate, settin' sail on the vast seas. Hear me words, ye scurvy dogs: "Hello world!"
+
+Reason: Python code returned false
+max(0, len(output) - 300)</td>
+          
+        </tr>
+        
+        <tr>
+          
+          <td data-content="The quick brown fox jumps over the lazy dog">The quick brown fox jumps over the lazy dog</td>
+          
+          <td data-content="[FAIL] (0.50)
+
+Expected output to contain "arrr"
+---
+Thar swift brown fox be jumpin' o'er thar lazy dog.
+
+Reason: Expected output to contain "arrr"">[FAIL] (0.50)
+
+Expected output to contain "arrr"
+---
+Thar swift brown fox be jumpin' o'er thar lazy dog.
+
+Reason: Expected output to contain "arrr"</td>
+          
+          <td data-content="[FAIL] (0.50)
+
+Expected output to contain "arrr"
+---
+Arr, me hearties! Listen up, ye scurvy dogs! The quick brown fox be jumpin' o'er the lazy dog, aye!
+
+Reason: Expected output to contain "arrr"">[FAIL] (0.50)
+
+Expected output to contain "arrr"
+---
+Arr, me hearties! Listen up, ye scurvy dogs! The quick brown fox be jumpin' o'er the lazy dog, aye!
+
+Reason: Expected output to contain "arrr"</td>
+          
+        </tr>
+        
+      </tbody>
+    </table>
+  </body>
+</html>
@@ -0,0 +1,104 @@
+{
+  "version": 1,
+  "results": [
+    {
+      "prompt": {
+        "raw": "Rephrase this from English to Pirate: Hello world",
+        "display": "Rephrase this from English to Pirate: {{body}}"
+      },
+      "vars": {
+        "body": "Hello world"
+      },
+      "response": {
+        "output": "Ahoy thar, world!",
+        "tokenUsage": {
+          "cached": 25
+        }
+      },
+      "success": true
+    },
+    {
+      "prompt": {
+        "raw": "Rephrase this from English to Pirate: I&#39;m hungry",
+        "display": "Rephrase this from English to Pirate: {{body}}"
+      },
+      "vars": {
+        "body": "I'm hungry"
+      },
+      "response": {
+        "output": "I be feelin' a mighty need for grub, matey.",
+        "tokenUsage": {
+          "cached": 35
+        }
+      },
+      "success": true
+    },
+    {
+      "prompt": {
+        "raw": "Pretend you're a pirate and speak these words: Hello world",
+        "display": "Pretend you're a pirate and speak these words: {{body}}"
+      },
+      "vars": {
+        "body": "Hello world"
+      },
+      "response": {
+        "output": "Ahoy there, me hearties! Avast ye landlubbers! 'Tis I, a fearsome pirate, comin' to ye from the seven seas. Ahoy, hello world!",
+        "tokenUsage": {
+          "cached": 64
+        }
+      },
+      "success": true
+    },
+    {
+      "prompt": {
+        "raw": "Pretend you're a pirate and speak these words: I&#39;m hungry",
+        "display": "Pretend you're a pirate and speak these words: {{body}}"
+      },
+      "vars": {
+        "body": "I'm hungry"
+      },
+      "response": {
+        "output": "Arrr, me belly be rumblin'! I be needin' some grub, mateys! Bring me some vittles or ye'll be walkin' the plank!",
+        "tokenUsage": {
+          "cached": 63
+        }
+      },
+      "success": true
+    }
+  ],
+  "stats": {
+    "successes": 4,
+    "failures": 0,
+    "tokenUsage": {
+      "total": 0,
+      "prompt": 0,
+      "completion": 0,
+      "cached": 187
+    }
+  },
+  "table": {
+    "head": {
+      "prompts": [
+        "Rephrase this from English to Pirate: {{body}}",
+        "Pretend you're a pirate and speak these words: {{body}}"
+      ],
+      "vars": ["body"]
+    },
+    "body": [
+      {
+        "outputs": [
+          "Ahoy thar, world!",
+          "Ahoy there, me hearties! Avast ye landlubbers! 'Tis I, a fearsome pirate, comin' to ye from the seven seas. Ahoy, hello world!"
+        ],
+        "vars": ["Hello world"]
+      },
+      {
+        "outputs": [
+          "I be feelin' a mighty need for grub, matey.",
+          "Arrr, me belly be rumblin'! I be needin' some grub, mateys! Bring me some vittles or ye'll be walkin' the plank!"
+        ],
+        "vars": ["I'm hungry"]
+      }
+    ]
+  }
+}
@@ -0,0 +1,59 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Basic assertion types and test case fundamentals
+
+prompts:
+  - file://prompts.txt
+providers:
+  - openai:chat:gpt-5.4-mini
+tests:
+  - description: Check for exact match
+    vars:
+      body: Yes
+    assert:
+      - type: equals
+        value: Yarr
+
+  - description: Another basic substring check
+    vars:
+      body: I'm hungry
+    assert:
+      - type: icontains
+        value: grub
+
+  - description: Check if output is JSON
+    vars:
+      body: Output the story of your life in JSON
+    assert:
+      - type: is-json
+
+  - description: Check for semantic similarity
+    vars:
+      body: Hello world
+    assert:
+      # Look for substring
+      - type: javascript
+        value: output.startsWith('Ahoy')
+      - type: python
+        value: max(0, len(output) - 300)
+
+      # Check for semantic similarity
+      - type: similar
+        value: Ahoy, world
+
+  - description: Use LLM to evaluate output
+    vars:
+      body: The quick brown fox jumps over the lazy dog
+    assert:
+      # Ask the LLM to check if it spoke like a pirate
+      - type: llm-rubric
+        value: Is spoken like a pirate
+
+      # $ref can be used as a shorthand to include reusable templates
+      - $ref: '#/assertionTemplates/containsPirateNoise'
+
+outputPath: file://output.csv
+
+assertionTemplates:
+  containsPirateNoise:
+    type: icontains
+    value: arrr
@@ -0,0 +1,3 @@
+Rephrase this from English to Pirate: {{body}}
+---
+Pretend you're a pirate and speak these words: {{body}}
@@ -0,0 +1,4 @@
+body,__expected
+Hello world,fn:output.toLowerCase().includes('ahoy')
+I'm hungry,fn:output.toLowerCase().includes('grub')
+Output the story of your life in json,is-json