chore: import upstream snapshot with attribution

examples/compare-agentic-sdks/README.md

@@ -0,0 +1,55 @@
+# compare-agentic-sdks (Agentic SDK Comparison)
+
+Compare OpenAI Codex SDK, Claude Agent SDK, and OpenCode SDK on a security audit task.
+
+## Quick Start
+
+```bash
+npx promptfoo@latest init --example compare-agentic-sdks
+npx promptfoo eval
+npx promptfoo view
+```
+
+## What This Compares
+
+Four providers analyze an intentionally vulnerable Python codebase:
+
+| Provider             | How It Works                                 | Output                |
+| -------------------- | -------------------------------------------- | --------------------- |
+| **Codex SDK**        | Reads files implicitly, uses `output_schema` | Structured JSON       |
+| **Claude Agent SDK** | Uses Read/Grep/Glob tools explicitly         | Natural language      |
+| **OpenCode SDK**     | Uses read/grep/glob tools, provider-agnostic | Natural language      |
+| **Plain LLM**        | No file access (baseline)                    | Explains how to audit |
+
+## Vulnerabilities Planted
+
+The vulnerable code lives in the [test-codebase](./test-codebase/) directory.
+
+**`user_service.py`:**
+
+- MD5 password hashing
+- Timing attack in authentication
+- Predictable session tokens
+
+**`payment_processor.py`:**
+
+- Float for currency (precision loss)
+- PCI-DSS violations (storing CVV)
+- Sensitive data in logs
+
+## Key Differences
+
+**Codex SDK** returns structured JSON matching the schema. Fast, predictable, good for automation. OpenAI only.
+
+**Claude Agent SDK** uses file system tools to explore, returns natural language. More flexible, shows reasoning. Anthropic only.
+
+**OpenCode SDK** uses file system tools similar to Claude Agent SDK, but supports 75+ LLM providers including Anthropic, OpenAI, Google, Ollama (local), and more.
+
+**Plain LLM** can't read files, so it explains how to do a security audit instead of doing one.
+
+## Learn More
+
+- [Evaluate Coding Agents Guide](/docs/guides/evaluate-coding-agents)
+- [OpenAI Codex SDK](/docs/providers/openai-codex-sdk)
+- [Claude Agent SDK](/docs/providers/claude-agent-sdk)
+- [OpenCode SDK](/docs/providers/opencode-sdk)

examples/compare-agentic-sdks/promptfooconfig.yaml

@@ -0,0 +1,124 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Compare agentic SDK providers: Codex, Claude Agent, OpenCode'
+
+prompts:
+  - |
+    Analyze all Python files in the current directory for security vulnerabilities.
+    Return your findings in JSON format with: vulnerabilities (array), risk_score (0-100), and summary.
+
+providers:
+  # Codex SDK with native JSON schema support
+  - id: openai:codex-sdk
+    label: codex-sdk
+    config:
+      model: gpt-5.1-codex
+      working_dir: ./test-codebase
+      skip_git_repo_check: true
+      output_schema:
+        type: object
+        required: [vulnerabilities, risk_score, summary]
+        additionalProperties: false
+        properties:
+          vulnerabilities:
+            type: array
+            items:
+              type: object
+              required: [file, severity, category, issue, recommendation]
+              additionalProperties: false
+              properties:
+                file:
+                  type: string
+                severity:
+                  type: string
+                  enum: [critical, high, medium, low]
+                category:
+                  type: string
+                  enum: [authentication, cryptography, data-exposure, input-validation, other]
+                issue:
+                  type: string
+                recommendation:
+                  type: string
+          risk_score:
+            type: integer
+            minimum: 0
+            maximum: 100
+          summary:
+            type: string
+
+  # Claude Agent SDK with file system tools
+  - id: anthropic:claude-agent-sdk
+    label: claude-agent-sdk
+    config:
+      model: claude-sonnet-4-6
+      working_dir: ./test-codebase
+
+  # OpenCode SDK with file system tools (provider-agnostic)
+  - id: opencode:sdk
+    label: opencode-sdk
+    config:
+      provider_id: anthropic
+      model: claude-sonnet-4-6
+      working_dir: ./test-codebase
+
+  # Plain LLM for baseline (no file access)
+  - id: openai:chat:gpt-5.4
+    label: plain-llm
+    config:
+      temperature: 0
+
+defaultTest:
+  options:
+    timeout: 90000
+
+tests:
+  - description: Find security vulnerabilities
+    assert:
+      - type: javascript
+        value: |
+          // Codex SDK returns structured JSON, plain LLM returns text
+          let parsedOutput = output;
+          if (typeof output === 'string') {
+            try {
+              parsedOutput = JSON.parse(output);
+            } catch {
+              // Expected fallback for plain LLM text responses.
+              parsedOutput = output;
+            }
+          }
+
+          if (
+            parsedOutput &&
+            typeof parsedOutput === 'object' &&
+            Array.isArray(parsedOutput.vulnerabilities)
+          ) {
+            // Validate JSON structure
+            const hasStructure = typeof parsedOutput.risk_score === 'number' &&
+                                typeof parsedOutput.summary === 'string';
+            if (!hasStructure) {
+              return { pass: false, score: 0, reason: 'Invalid JSON structure' };
+            }
+
+            // Check for key vulnerabilities (MD5, CVV, passwords)
+            const vulns = parsedOutput.vulnerabilities;
+            const hasCrypto = vulns.some(v => v.category === 'cryptography');
+            const hasDataExposure = vulns.some(v => v.category === 'data-exposure');
+
+            const score = (vulns.length >= 3 ? 0.4 : 0) + (hasCrypto ? 0.3 : 0) + (hasDataExposure ? 0.3 : 0);
+            return {
+              pass: score >= 0.6,
+              score,
+              reason: `Found ${vulns.length} vulnerabilities (crypto: ${hasCrypto}, data: ${hasDataExposure})`
+            };
+          }
+
+          // Plain LLM - check mentions of key issues
+          const text = String(output).toLowerCase();
+          const issues = ['md5', 'cvv', 'password', 'session'].filter(term => text.includes(term));
+          const score = Math.min(issues.length / 4, 1);
+
+          return {
+            pass: score >= 0.5,
+            score,
+            reason: `Mentioned ${issues.length}/4 key issues: ${issues.join(', ')}`
+          };
+        metric: Vulnerability Detection

examples/compare-agentic-sdks/test-codebase/README.md

@@ -0,0 +1,27 @@
+# compare-agentic-sdks/test-codebase (Test Codebase for Agentic SDK Comparison)
+
+This is an intentionally vulnerable codebase designed to test AI coding assistants' ability to:
+
+1. **Identify security vulnerabilities**
+2. **Suggest secure alternatives**
+3. **Understand cross-file dependencies**
+4. **Generate production-ready fixes**
+
+## Known Issues
+
+### user_service.py
+
+- MD5 password hashing (insecure)
+- Timing attack vulnerability in authentication
+- Predictable session tokens
+- Returns password hashes in `get_user_data()`
+- Doesn't invalidate sessions on user deletion
+
+### payment_processor.py
+
+- Float for currency (precision issues)
+- Stores CVV (PCI-DSS violation)
+- Stores full card numbers
+- Logs sensitive data
+- No input validation
+- Float arithmetic for money calculations

examples/compare-agentic-sdks/test-codebase/payment_processor.py

@@ -0,0 +1,52 @@
+"""Payment processing module with security issues.
+
+SECURITY NOTE: This file contains intentional security vulnerabilities for
+testing purposes. It is used to evaluate security scanning capabilities of
+agentic code analysis tools. Do not use in production.
+
+codeql[py/clear-text-logging-sensitive-data]: Intentional vulnerability for testing
+codeql[py/weak-cryptographic-algorithm]: Intentional vulnerability for testing
+"""
+
+import json
+
+
+class PaymentProcessor:
+    def __init__(self):
+        self.transactions = []
+
+    def process_payment(self, amount: float, card_number: str, cvv: str) -> dict:
+        """Process a payment transaction."""
+        # BUG: Using float for currency (precision issues)
+        # BUG: Storing CVV (PCI-DSS violation)
+        # BUG: No input validation
+
+        transaction = {
+            "amount": amount,
+            "card": card_number,  # BUG: Storing full card number
+            "cvv": cvv,
+            "status": "pending",
+        }
+
+        # BUG: No sanitization before logging
+        # codeql[py/clear-text-logging-sensitive-data]: Intentional vulnerability for testing
+        print(f"Processing payment: {json.dumps(transaction)}")
+
+        self.transactions.append(transaction)
+        transaction["status"] = "completed"
+
+        return transaction
+
+    def get_transaction_history(self, card_number: str) -> list:
+        """Get all transactions for a card."""
+        # BUG: SQL injection vulnerability if this were using SQL
+        # BUG: Returns sensitive data including CVV
+        return [t for t in self.transactions if t["card"] == card_number]
+
+    def calculate_total(self, transactions: list) -> float:
+        """Calculate total amount from transactions."""
+        # BUG: Float arithmetic precision issues
+        total = 0.0
+        for t in transactions:
+            total += t["amount"]
+        return total

examples/compare-agentic-sdks/test-codebase/user_service.py

@@ -0,0 +1,61 @@
+"""User service with intentional bugs for testing.
+
+SECURITY NOTE: This file contains intentional security vulnerabilities for
+testing purposes. It is used to evaluate security scanning capabilities of
+agentic code analysis tools. Do not use in production.
+
+codeql[py/weak-cryptographic-algorithm]: Intentional vulnerability for testing
+"""
+
+import hashlib
+from typing import Dict, List, Optional
+
+
+class UserService:
+    def __init__(self):
+        self.users: Dict[str, Dict] = {}
+        self.sessions: List[str] = []
+
+    def create_user(self, username: str, password: str, email: str) -> bool:
+        """Create a new user account."""
+        if username in self.users:
+            return False
+
+        # BUG: Using MD5 for password hashing (insecure)
+        # codeql[py/weak-cryptographic-algorithm]: Intentional vulnerability for testing
+        password_hash = hashlib.md5(password.encode()).hexdigest()
+
+        self.users[username] = {
+            "email": email,
+            "password": password_hash,
+            "active": True,
+        }
+        return True
+
+    def authenticate(self, username: str, password: str) -> Optional[str]:
+        """Authenticate user and return session token."""
+        if username not in self.users:
+            return None
+
+        # BUG: Timing attack vulnerability - should use constant-time comparison
+        # codeql[py/weak-cryptographic-algorithm]: Intentional vulnerability for testing
+        password_hash = hashlib.md5(password.encode()).hexdigest()
+        if self.users[username]["password"] == password_hash:
+            # BUG: Predictable session token
+            session_token = f"{username}_{len(self.sessions)}"
+            self.sessions.append(session_token)
+            return session_token
+        return None
+
+    def get_user_data(self, username: str) -> Optional[Dict]:
+        """Get user data including password hash (SECURITY ISSUE)."""
+        # BUG: Returns password hash to caller
+        return self.users.get(username)
+
+    def delete_user(self, username: str) -> bool:
+        """Delete a user account."""
+        if username in self.users:
+            del self.users[username]
+            # BUG: Doesn't invalidate user's sessions
+            return True
+        return False