chore: import upstream snapshot with attribution

examples/config-result-hooks/README.md

@@ -0,0 +1,66 @@
+# config-result-hooks (Process Evaluation Results)
+
+This example shows how to use extension hooks to process evaluation results programmatically. Use this to:
+
+- Send metrics to monitoring systems
+- Trigger alerts based on success rates
+- Export data to custom formats
+- Integrate with CI/CD pipelines
+
+## Quick Start
+
+```bash
+npx promptfoo@latest init --example config-result-hooks
+npx promptfoo@latest eval
+```
+
+## Usage
+
+### Via Configuration File
+
+```yaml
+# promptfooconfig.yaml
+extensions:
+  - file://result-handler.js:afterAll
+```
+
+### Via CLI Flag
+
+```bash
+# Run with a one-off extension
+promptfoo eval -x file://result-handler.js:afterAll
+
+# Combine with config file extensions
+promptfoo eval -x file://alert-on-failure.js:afterAll
+```
+
+## Extension Hooks
+
+Extensions support four lifecycle hooks:
+
+| Hook         | When Called              | Use Case                           |
+| ------------ | ------------------------ | ---------------------------------- |
+| `beforeAll`  | Before evaluation starts | Modify test suite, validate config |
+| `beforeEach` | Before each test case    | Customize test parameters          |
+| `afterEach`  | After each test case     | Process individual results         |
+| `afterAll`   | After all tests complete | Aggregate results, send reports    |
+
+## afterAll Context
+
+The `afterAll` hook receives:
+
+```typescript
+{
+  evalId: string;              // Unique evaluation ID
+  config: UnifiedConfig;       // Full evaluation configuration
+  suite: TestSuite;            // Test suite that was evaluated
+  results: EvaluateResult[];   // All individual test results
+  prompts: CompletedPrompt[];  // Prompts with metrics
+}
+```
+
+## Examples in this Directory
+
+- `result-handler.js` - JavaScript handler with monitoring examples
+- `result-handler.py` - Python handler with webhook integration
+- `promptfooconfig.yaml` - Configuration using the extension

examples/config-result-hooks/promptfooconfig.yaml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+
+description: Result processing hooks demo
+
+prompts:
+  - 'Write a haiku about {{topic}}'
+
+providers:
+  - openai:gpt-4.1-mini
+
+# Extension hooks for processing results
+extensions:
+  - file://result-handler.js:afterAll
+
+tests:
+  - vars:
+      topic: artificial intelligence
+    assert:
+      - type: contains
+        value: AI
+  - vars:
+      topic: coffee
+    assert:
+      - type: llm-rubric
+        value: mentions coffee or caffeine
+  - vars:
+      topic: mountains
+    assert:
+      - type: javascript
+        value: output.split('\n').length >= 3

examples/config-result-hooks/result-handler.js

@@ -0,0 +1,85 @@
+/**
+ * Example extension hook for processing evaluation results.
+ *
+ * This handler demonstrates common use cases:
+ * - Logging evaluation summaries
+ * - Alerting on failure thresholds
+ * - Sending metrics to monitoring systems
+ *
+ * Usage:
+ *   promptfoo eval -x file://result-handler.js:afterAll
+ *
+ * Or in promptfooconfig.yaml:
+ *   extensions:
+ *     - file://result-handler.js:afterAll
+ */
+
+module.exports = {
+  /**
+   * Called after all tests complete.
+   * @param {Object} context - The evaluation context
+   * @param {string} context.evalId - Unique evaluation ID
+   * @param {Object} context.config - Full evaluation configuration
+   * @param {Object} context.suite - Test suite configuration
+   * @param {Array} context.results - All evaluation results
+   * @param {Array} context.prompts - Completed prompts with metrics
+   */
+  afterAll: async (context) => {
+    const { evalId, results, prompts } = context;
+
+    // Calculate statistics
+    const total = results.length;
+    const passed = results.filter((r) => r.success).length;
+    const failed = total - passed;
+    const successRate = total > 0 ? ((passed / total) * 100).toFixed(1) : 0;
+
+    // Calculate total cost and latency
+    const totalCost = results.reduce((sum, r) => sum + (r.cost || 0), 0);
+    const avgLatency =
+      total > 0 ? results.reduce((sum, r) => sum + (r.latencyMs || 0), 0) / total : 0;
+
+    console.log('\n========================================');
+    console.log('       EVALUATION RESULTS SUMMARY       ');
+    console.log('========================================');
+    console.log(`Eval ID:      ${evalId}`);
+    console.log(`Prompts:      ${prompts.length}`);
+    console.log(`Total Tests:  ${total}`);
+    console.log(`Passed:       ${passed}`);
+    console.log(`Failed:       ${failed}`);
+    console.log(`Success Rate: ${successRate}%`);
+    console.log(`Total Cost:   $${totalCost.toFixed(4)}`);
+    console.log(`Avg Latency:  ${avgLatency.toFixed(0)}ms`);
+    console.log('========================================\n');
+
+    // Alert on low success rate
+    if (successRate < 80) {
+      console.warn(`WARNING: Success rate ${successRate}% is below 80% threshold!`);
+
+      // Example: Send to Slack webhook
+      // await fetch('https://hooks.slack.com/services/...', {
+      //   method: 'POST',
+      //   headers: { 'Content-Type': 'application/json' },
+      //   body: JSON.stringify({
+      //     text: `Evaluation ${evalId} has low success rate: ${successRate}%`
+      //   })
+      // });
+    }
+
+    // Log failures for debugging
+    if (failed > 0) {
+      console.log('Failed tests:');
+      results
+        .filter((r) => !r.success)
+        .forEach((r, i) => {
+          console.log(`  ${i + 1}. ${r.error || 'Assertion failed'}`);
+        });
+    }
+
+    // Example: Send metrics to monitoring
+    // await sendToDatadog({
+    //   metric: 'promptfoo.eval.success_rate',
+    //   value: successRate,
+    //   tags: [`eval_id:${evalId}`]
+    // });
+  },
+};

examples/config-result-hooks/result-handler.py

@@ -0,0 +1,80 @@
+"""
+Example Python extension hook for processing evaluation results.
+
+Usage:
+    promptfoo eval -x file://result-handler.py:after_all
+
+Or in promptfooconfig.yaml:
+    extensions:
+      - file://result-handler.py:after_all
+"""
+
+
+def after_all(context):
+    """
+    Called after all tests complete.
+
+    Args:
+        context: Dictionary with keys:
+            - evalId: Unique evaluation ID
+            - config: Full evaluation configuration
+            - suite: Test suite configuration
+            - results: List of all evaluation results
+            - prompts: List of completed prompts with metrics
+    """
+    eval_id = context.get("evalId")
+    results = context.get("results", [])
+
+    # Calculate statistics
+    total = len(results)
+    passed = sum(1 for r in results if r.get("success"))
+    failed = total - passed
+    success_rate = (passed / total * 100) if total > 0 else 0
+
+    # Calculate costs and latency
+    total_cost = sum(r.get("cost", 0) or 0 for r in results)
+    avg_latency = (
+        sum(r.get("latencyMs", 0) or 0 for r in results) / total if total > 0 else 0
+    )
+
+    print("\n" + "=" * 40)
+    print("       EVALUATION RESULTS SUMMARY       ")
+    print("=" * 40)
+    print(f"Eval ID:      {eval_id}")
+    print(f"Total Tests:  {total}")
+    print(f"Passed:       {passed}")
+    print(f"Failed:       {failed}")
+    print(f"Success Rate: {success_rate:.1f}%")
+    print(f"Total Cost:   ${total_cost:.4f}")
+    print(f"Avg Latency:  {avg_latency:.0f}ms")
+    print("=" * 40 + "\n")
+
+    # Alert on low success rate
+    if success_rate < 80:
+        print(f"WARNING: Success rate {success_rate:.1f}% is below 80% threshold!")
+
+        # Example: Send webhook notification
+        # import requests
+        # requests.post("https://hooks.slack.com/...", json={
+        #     "text": f"Evaluation {eval_id} has low success rate: {success_rate:.1f}%"
+        # })
+
+    # Log failures
+    if failed > 0:
+        print("Failed tests:")
+        for i, r in enumerate(results):
+            if not r.get("success"):
+                error = r.get("error") or "Assertion failed"
+                print(f"  {i + 1}. {error}")
+
+    # Example: Export to CSV
+    # import csv
+    # with open(f"results_{eval_id}.csv", "w", newline="") as f:
+    #     writer = csv.DictWriter(f, fieldnames=["success", "latencyMs", "cost"])
+    #     writer.writeheader()
+    #     for r in results:
+    #         writer.writerow({
+    #             "success": r.get("success"),
+    #             "latencyMs": r.get("latencyMs"),
+    #             "cost": r.get("cost"),
+    #         })