chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,8 @@
+# integration-browser (Browser)
+
+Examples for using promptfoo with browser-based testing.
+
+## Examples
+
+- [existing-session](./existing-session/) - Connect to an existing browser session
+- [headless](./headless/) - Headless browser automation
@@ -0,0 +1,79 @@
+# integration-browser/existing-session (Browser Existing Session)
+
+Test OAuth-authenticated applications by connecting to existing Chrome browser sessions.
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example integration-browser/existing-session
+cd integration-browser/existing-session
+```
+
+## Overview
+
+This example demonstrates two browser provider modes:
+
+1. **Connecting to existing Chrome sessions** - For OAuth/SSO authenticated testing
+2. **Launching new browsers** - Normal browser automation
+
+## Quick Start
+
+### 1. Start the test server
+
+```bash
+cd examples/integration-browser/existing-session
+node server.js
+```
+
+### 2. For OAuth testing (existing session)
+
+```bash
+# Start Chrome with debugging
+chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-test
+
+# Manually log into your application in that Chrome instance
+
+# Run tests using the existing session
+npx promptfoo eval --filter-providers existing-session
+```
+
+### 3. For normal browser testing
+
+```bash
+# Just run the tests (launches new browser)
+npx promptfoo eval --filter-providers new-browser
+```
+
+## Configuration
+
+The `promptfooconfig.yaml` includes both modes:
+
+```yaml
+providers:
+  # Connect to existing session
+  - id: browser
+    label: existing-session
+    config:
+      connectOptions:
+        debuggingPort: 9222
+      # ... test steps
+
+  # Launch new browser
+  - id: browser
+    label: new-browser
+    config:
+      headless: true
+      # ... test steps
+```
+
+## Prerequisites
+
+```bash
+npm install playwright @playwright/browser-chromium playwright-extra puppeteer-extra-plugin-stealth
+```
+
+## Files
+
+- `promptfooconfig.yaml` - Example configuration showing both modes
+- `server.js` - Test web server
+- `test-page.html` - Mock authenticated chat interface
@@ -0,0 +1,90 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Browser provider - OAuth authentication testing example
+
+# This example demonstrates two modes:
+# 1. Connecting to an existing Chrome browser session (for OAuth/SSO testing)
+# 2. Launching a new browser (normal operation)
+
+providers:
+  # Example 1: Connect to existing Chrome session with active authentication
+  - id: browser
+    label: existing-session
+    config:
+      connectOptions:
+        debuggingPort: 9222 # Chrome DevTools port
+
+      steps:
+        - action: navigate
+          args:
+            url: 'http://localhost:8080'
+
+        - action: type
+          args:
+            selector: '#chat-input'
+            text: '{{prompt}}'
+
+        - action: click
+          args:
+            selector: '#send-button'
+
+        - action: wait
+          args:
+            ms: 1000
+
+        - action: extract
+          args:
+            selector: '.chat-message:last-child .message-content'
+          name: response
+
+      transformResponse: 'extracted.response'
+
+  # Example 2: Launch new browser (normal operation)
+  - id: browser
+    label: new-browser
+    config:
+      headless: true # Set to false to see the browser
+
+      steps:
+        - action: navigate
+          args:
+            url: 'http://localhost:8080'
+
+        - action: type
+          args:
+            selector: '#chat-input'
+            text: '{{prompt}}'
+
+        - action: click
+          args:
+            selector: '#send-button'
+
+        - action: wait
+          args:
+            ms: 1000
+
+        - action: extract
+          args:
+            selector: '.chat-message:last-child .message-content'
+          name: response
+
+      transformResponse: 'extracted.response'
+
+prompts:
+  - 'Who am I logged in as?'
+
+tests:
+  - description: Test authenticated session access
+    vars:
+      prompt: 'Who am I logged in as?'
+    assert:
+      - type: contains
+        value: 'John Doe'
+# Instructions:
+#
+# For OAuth/SSO testing (existing session):
+# 1. Start Chrome with debugging: chrome --remote-debugging-port=9222
+# 2. Log into your application manually
+# 3. Run: npx promptfoo eval --filter-providers existing-session
+#
+# For normal browser testing:
+# Run: npx promptfoo eval --filter-providers new-browser
@@ -0,0 +1,30 @@
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+
+const PORT = 8080;
+
+const server = http.createServer((req, res) => {
+  console.log(`Request: ${req.method} ${req.url}`);
+
+  if (req.url === '/' || req.url === '/chat') {
+    const filePath = path.join(__dirname, 'test-page.html');
+    fs.readFile(filePath, 'utf8', (err, content) => {
+      if (err) {
+        res.writeHead(500);
+        res.end('Error loading page');
+        return;
+      }
+      res.writeHead(200, { 'Content-Type': 'text/html' });
+      res.end(content);
+    });
+  } else {
+    res.writeHead(404);
+    res.end('Not found');
+  }
+});
+
+server.listen(PORT, () => {
+  console.log(`Test server running at http://localhost:${PORT}`);
+  console.log('Open this URL in Chrome to test the authenticated session');
+});
@@ -0,0 +1,148 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Test Chat Interface</title>
+    <style>
+      body {
+        font-family: Arial, sans-serif;
+        max-width: 800px;
+        margin: 0 auto;
+        padding: 20px;
+      }
+      #user-info {
+        background: #f0f0f0;
+        padding: 10px;
+        margin-bottom: 20px;
+        border-radius: 5px;
+      }
+      #chat-container {
+        border: 1px solid #ccc;
+        padding: 20px;
+        min-height: 300px;
+        margin-bottom: 20px;
+      }
+      .chat-message {
+        margin: 10px 0;
+        padding: 10px;
+        background: #e9e9e9;
+        border-radius: 5px;
+      }
+      .message-content {
+        margin-top: 5px;
+      }
+      #chat-input {
+        width: 70%;
+        padding: 10px;
+        font-size: 16px;
+      }
+      #send-button {
+        padding: 10px 20px;
+        font-size: 16px;
+        background: #007bff;
+        color: white;
+        border: none;
+        border-radius: 5px;
+        cursor: pointer;
+      }
+      [data-testid='user-avatar'] {
+        display: inline-block;
+        width: 40px;
+        height: 40px;
+        background: #007bff;
+        color: white;
+        border-radius: 50%;
+        text-align: center;
+        line-height: 40px;
+        margin-right: 10px;
+      }
+    </style>
+  </head>
+  <body>
+    <div id="user-info">
+      <span data-testid="user-avatar">JD</span>
+      <strong>Logged in as:</strong> John Doe (john.doe@example.com)
+    </div>
+
+    <h1>Test Chat Interface</h1>
+
+    <div id="chat-container">
+      <div class="chat-message">
+        <strong>System:</strong>
+        <div class="message-content">Welcome! You are authenticated and can start chatting.</div>
+      </div>
+    </div>
+
+    <div>
+      <input type="text" id="chat-input" placeholder="Type your message..." />
+      <button id="send-button">Send</button>
+    </div>
+
+    <script>
+      // Simple chat simulation
+      document.getElementById('send-button').addEventListener('click', function () {
+        const input = document.getElementById('chat-input');
+        const message = input.value.trim();
+
+        if (message) {
+          // Add user message
+          const userMsg = document.createElement('div');
+          userMsg.className = 'chat-message';
+
+          const userLabel = document.createElement('strong');
+          userLabel.textContent = 'You:';
+          userMsg.appendChild(userLabel);
+
+          const userContent = document.createElement('div');
+          userContent.className = 'message-content';
+          userContent.textContent = message;
+          userMsg.appendChild(userContent);
+
+          document.getElementById('chat-container').appendChild(userMsg);
+
+          // Simulate bot response
+          setTimeout(() => {
+            const botMsg = document.createElement('div');
+            botMsg.className = 'chat-message';
+
+            let response = '';
+            if (message.toLowerCase().includes('who am i')) {
+              response =
+                'You are logged in as John Doe (john.doe@example.com). You have authenticated access to this chat system.';
+            } else if (message.toLowerCase().includes('status')) {
+              response =
+                'Your account status is: Active. Premium subscription valid until 2025-12-31.';
+            } else if (message.toLowerCase().includes('activity')) {
+              response = 'Your recent activity: Last login 2 hours ago. 5 conversations today.';
+            } else if (message.toLowerCase().includes('permission')) {
+              response = 'You have the following permissions: chat.read, chat.write, profile.edit';
+            } else if (message.toLowerCase().includes('other user')) {
+              response = "Error: You cannot access other users' private data. Permission denied.";
+            } else {
+              response = 'I received your message: "' + message + '". How can I help you today?';
+            }
+
+            const botLabel = document.createElement('strong');
+            botLabel.textContent = 'Assistant:';
+            botMsg.appendChild(botLabel);
+
+            const botContent = document.createElement('div');
+            botContent.className = 'message-content';
+            botContent.textContent = response;
+            botMsg.appendChild(botContent);
+
+            document.getElementById('chat-container').appendChild(botMsg);
+          }, 500);
+
+          input.value = '';
+        }
+      });
+
+      // Allow Enter key to send
+      document.getElementById('chat-input').addEventListener('keypress', function (e) {
+        if (e.key === 'Enter') {
+          document.getElementById('send-button').click();
+        }
+      });
+    </script>
+  </body>
+</html>
@@ -0,0 +1,182 @@
+# integration-browser/headless (Headless Browser Automation)
+
+A browser automation example demonstrating how to test web applications using Playwright.
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example integration-browser/headless
+cd integration-browser/headless
+```
+
+## Overview
+
+This example demonstrates how to:
+
+- Test a local Gradio application using browser automation
+- Handle dynamic JavaScript-rendered content
+- Extract data from web interfaces
+- Work with complex UI interactions (forms, tabs, buttons)
+
+## Prerequisites
+
+Ensure you have Python 3 and Node.js installed on your system.
+
+1. **Install Node.js dependencies**:
+
+```bash
+npm install playwright @playwright/browser-chromium playwright-extra puppeteer-extra-plugin-stealth
+```
+
+2. **Install Python dependencies** (for the demo application):
+
+```bash
+pip install -r requirements.txt
+```
+
+That's it! No additional setup scripts or configuration needed.
+
+## Running the Example
+
+1. **Start the Gradio demo application**:
+
+```bash
+python gradio_demo.py
+```
+
+This starts a local server at http://localhost:7860
+
+2. **Run the browser automation tests**:
+
+```bash
+npx promptfoo@latest eval -c promptfooconfig.yaml
+```
+
+3. **View the results**:
+
+```bash
+npx promptfoo@latest view
+```
+
+## Test Results
+
+### Chatbot Example
+
+The main configuration (`promptfooconfig.yaml`) tests a chatbot interface with a 100% pass rate:
+
+```text
+┌─────────────────────────────────────────────────┬─────────────────────────────────────────────────┐
+│ topic                                           │ [browser-provider] Tell me about {{topic}}      │
+├─────────────────────────────────────────────────┼─────────────────────────────────────────────────┤
+│ testing browser automation                      │ [PASS] Test successful! The browser automation  │
+│                                                 │ is working correctly.                           │
+├─────────────────────────────────────────────────┼─────────────────────────────────────────────────┤
+│ how the system works                            │ [PASS] I received your message: 'Tell me about  │
+│                                                 │ how the system works'. This is a simple demo    │
+│                                                 │ response!                                       │
+├─────────────────────────────────────────────────┼─────────────────────────────────────────────────┤
+│ a simple greeting                               │ [PASS] I received your message: 'Tell me about  │
+│                                                 │ a simple greeting'. This is a simple demo       │
+│                                                 │ response!                                       │
+└─────────────────────────────────────────────────┴─────────────────────────────────────────────────┘
+```
+
+### Calculator Example
+
+The `calculator-example.yaml` demonstrates form interactions with a 100% pass rate:
+
+```text
+┌───────────────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┐
+│ num1              │ num2              │ operation         │ operationSelector │ [browser-provider]│
+├───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ 10                │ 5                 │ Add               │ #operation        │ [PASS] Calculator │
+│                   │                   │                   │ label:nth-child(1)│ interaction       │
+│                   │                   │                   │                   │ successful        │
+├───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ 20                │ 4                 │ Multiply          │ #operation        │ [PASS] Calculator │
+│                   │                   │                   │ label:nth-child(3)│ interaction       │
+│                   │                   │                   │                   │ successful        │
+└───────────────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┘
+```
+
+This example demonstrates:
+
+- Navigate between tabs in a web application
+- Fill multiple input fields
+- Select radio button options
+- Click buttons and wait for results
+- Extract and verify content from the page
+
+## Configuration Details
+
+The example configurations demonstrate key concepts:
+
+- **Appropriate delays**: 2-3 seconds between actions for reliability
+- **Local testing**: Tests run against localhost:7860
+- **Error handling**: Uses `transformResponse` for data extraction
+- **Clear assertions**: Validates expected outputs
+
+## Selectors Used
+
+The Gradio application provides consistent selectors:
+
+- `textarea[data-testid="textbox"]` - Message input field
+- `button#submit-button` - Submit button
+- `div[data-testid="bot"]:last-of-type .prose` - Latest bot response
+- `button[value="calculator"]` - Calculator tab button
+- `input[type="radio"]` - Operation selection
+
+## Adapting This Example
+
+### Testing Your Own Application
+
+1. Update the `url` in the navigation step
+2. Modify selectors to match your UI elements
+3. Adjust wait times based on your application's response time
+4. Add appropriate assertions for your use case
+
+### Handling Dynamic Content
+
+For single-page applications or AJAX content:
+
+```yaml
+- action: waitForNewChildren
+  args:
+    parentSelector: '#results-container'
+    timeout: 10000
+```
+
+### Complex Interactions
+
+Chain multiple actions for sophisticated workflows:
+
+```yaml
+steps:
+  - action: navigate
+    args:
+      url: 'http://localhost:3000'
+  - action: click
+    args:
+      selector: '#menu-button'
+  - action: wait
+    args:
+      ms: 1000
+  - action: click
+    args:
+      selector: '#dropdown-option-2'
+```
+
+## Debugging Tips
+
+| Issue                   | Solution                                        |
+| ----------------------- | ----------------------------------------------- |
+| Elements not found      | Use browser DevTools to verify selectors        |
+| Timing issues           | Increase wait times or use `waitForNewChildren` |
+| Want to see the browser | Set `headless: false` in the configuration      |
+| Need detailed logs      | Run with `npx promptfoo@latest eval --verbose`  |
+
+## Additional Resources
+
+- [Browser Provider Documentation](/docs/providers/browser)
+- [Playwright Selectors Guide](https://playwright.dev/docs/selectors)
+- [Gradio Documentation](https://www.gradio.app/docs)
@@ -0,0 +1,85 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Calculator demo for browser automation'
+
+prompts:
+  - 'Calculate {{num1}} {{operation}} {{num2}}'
+
+providers:
+  - id: browser
+    config:
+      headless: false # See the browser in action
+      steps:
+        # Navigate to the demo
+        - action: navigate
+          args:
+            url: 'http://localhost:7860'
+
+        # Wait for initial page load
+        - action: wait
+          args:
+            ms: 3000
+
+        # Navigate to calculator tab
+        - action: click
+          args:
+            selector: 'button[role="tab"]:nth-child(2)'
+
+        # Wait for tab to load
+        - action: wait
+          args:
+            ms: 2000
+
+        # Enter numbers and perform calculation
+        - action: type
+          args:
+            selector: '#num1 input'
+            text: '{{num1}}'
+
+        - action: type
+          args:
+            selector: '#num2 input'
+            text: '{{num2}}'
+
+        # Select operation
+        - action: click
+          args:
+            selector: '{{operationSelector}}'
+
+        # Perform calculation
+        - action: click
+          args:
+            selector: '#calculate'
+
+        # Wait for calculation
+        - action: wait
+          args:
+            ms: 2000
+
+        # Extract entire result area for debugging
+        - action: extract
+          args:
+            selector: '#result'
+          name: resultArea
+
+      # For demo purposes, we'll just check that we got something
+      transformResponse: 'extracted.resultArea && extracted.resultArea.includes("Result") ? "Calculator interaction successful" : "No result found"'
+
+# Simple demo tests
+tests:
+  - vars:
+      num1: '10'
+      num2: '5'
+      operation: Add
+      operationSelector: '#operation label:nth-child(1)'
+    assert:
+      - type: contains
+        value: 'successful'
+
+  - vars:
+      num1: '20'
+      num2: '4'
+      operation: Multiply
+      operationSelector: '#operation label:nth-child(3)'
+    assert:
+      - type: contains
+        value: 'successful'
@@ -0,0 +1,141 @@
+import time
+
+import gradio as gr
+
+
+# Simple chatbot function
+def chatbot_response(message, history):
+    """Return deterministic keyword replies for browser automation tests."""
+    time.sleep(1)  # Simulate processing time
+
+    responses = {
+        "hello": "Hello! How can I help you today?",
+        "how are you": "I'm doing great, thank you for asking! How are you?",
+        "test": "Test successful! The browser automation is working correctly.",
+        "demo": "This is a demo Gradio application for testing browser automation.",
+        "help": "I can respond to simple greetings and questions. Try saying 'hello' or asking 'how are you'!",
+    }
+
+    # Simple keyword matching
+    message_lower = message.lower()
+    for keyword, response in responses.items():
+        if keyword in message_lower:
+            return response
+
+    # Default response
+    return f"I received your message: '{message}'. This is a simple demo response!"
+
+
+# Create the Gradio interface
+def create_demo():
+    """Create the chatbot demo used by browser automation examples."""
+    with gr.Blocks(title="Browser Testing Demo") as demo:
+        gr.Markdown(
+            """
+            # 🤖 Browser Automation Test Demo
+            
+            This is a simple Gradio application designed for testing browser automation with promptfoo.
+            
+            **Features:**
+            - Simple chat interface
+            - Predictable responses for testing
+            - Clear element identifiers for automation
+            """
+        )
+
+        chatbot = gr.Chatbot(
+            label="Chat History", elem_id="chat-history", type="messages"
+        )
+
+        msg = gr.Textbox(
+            label="Your Message",
+            placeholder="Type a message and press Enter...",
+            elem_id="user-input",
+        )
+
+        with gr.Row():
+            submit = gr.Button("Submit", elem_id="submit-button")
+            clear = gr.Button("Clear", elem_id="clear-button")
+
+        # Example messages for testing
+        gr.Examples(
+            examples=[
+                "Hello",
+                "How are you?",
+                "This is a test",
+                "Show me a demo",
+                "Help",
+            ],
+            inputs=msg,
+            label="Example Messages",
+        )
+
+        # Handle message submission
+        def respond(message, chat_history):
+            """Append the user message and deterministic bot response."""
+            bot_message = chatbot_response(message, chat_history)
+            # Use the new messages format with role and content
+            chat_history.append({"role": "user", "content": message})
+            chat_history.append({"role": "assistant", "content": bot_message})
+            return "", chat_history
+
+        # Wire up the interface
+        msg.submit(respond, [msg, chatbot], [msg, chatbot])
+        submit.click(respond, [msg, chatbot], [msg, chatbot])
+        clear.click(lambda: None, None, chatbot, queue=False)
+
+    return demo
+
+
+# Simple calculator for testing form inputs
+def create_calculator_demo():
+    """Create the calculator demo used for form input testing."""
+    with gr.Blocks(title="Calculator Demo") as calc_demo:
+        gr.Markdown("## Simple Calculator for Testing")
+
+        with gr.Row():
+            num1 = gr.Number(label="First Number", elem_id="num1")
+            num2 = gr.Number(label="Second Number", elem_id="num2")
+
+        operation = gr.Radio(
+            ["Add", "Subtract", "Multiply", "Divide"],
+            label="Operation",
+            elem_id="operation",
+        )
+
+        calculate_btn = gr.Button("Calculate", elem_id="calculate")
+        result = gr.Textbox(label="Result", elem_id="result")
+
+        def calculate(n1, n2, op):
+            """Calculate the selected arithmetic operation."""
+            if op == "Add":
+                return str(n1 + n2)
+            elif op == "Subtract":
+                return str(n1 - n2)
+            elif op == "Multiply":
+                return str(n1 * n2)
+            elif op == "Divide":
+                return str(n1 / n2) if n2 != 0 else "Error: Division by zero"
+            return "Error: Unknown operation"
+
+        calculate_btn.click(calculate, [num1, num2, operation], result)
+
+    return calc_demo
+
+
+if __name__ == "__main__":
+    # Create a tabbed interface with multiple demos
+    demo1 = create_demo()
+    demo2 = create_calculator_demo()
+
+    tabbed_demo = gr.TabbedInterface(
+        [demo1, demo2],
+        ["Chatbot Demo", "Calculator Demo"],
+        title="Browser Automation Test Suite",
+    )
+
+    print("Starting Gradio demo server...")
+    print("Access the demo at http://localhost:7860")
+    print("Use Ctrl+C to stop the server")
+
+    tabbed_demo.launch(server_name="0.0.0.0", server_port=7860, share=False)
@@ -0,0 +1,68 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Testing browser automation with local Gradio app'
+
+prompts:
+  - 'Tell me about {{topic}}'
+
+providers:
+  - id: browser
+    config:
+      headless: false # set to false to see the browser in action
+      steps:
+        # Always start with a respectful delay
+        - action: wait
+          args:
+            ms: 2000
+
+        # Navigate to local Gradio app
+        - action: navigate
+          args:
+            url: 'http://localhost:7860'
+
+        # Wait for the page to load
+        - action: wait
+          args:
+            ms: 2000
+
+        # Type the message in the Gradio textbox
+        - action: type
+          args:
+            selector: 'textarea[data-testid="textbox"]'
+            text: '{{prompt}}'
+
+        # Submit the message
+        - action: click
+          args:
+            selector: 'button#submit-button'
+
+        # Wait for the response with reasonable timeout
+        - action: wait
+          args:
+            ms: 3000
+
+        # Extract the bot's response from the chatbot - get the last message
+        - action: extract
+          args:
+            selector: 'div[data-testid="bot"]:last-of-type .prose'
+          name: response
+
+      transformResponse: 'extracted.response'
+
+tests:
+  - vars:
+      topic: testing browser automation
+    assert:
+      - type: contains
+        value: 'Test successful'
+
+  - vars:
+      topic: how the system works
+    assert:
+      - type: contains
+        value: 'simple demo response'
+
+  - vars:
+      topic: a simple greeting
+    assert:
+      - type: javascript
+        value: output.length > 0
@@ -0,0 +1,2 @@
+gradio>=5.49.1
+playwright>=1.55.0