chore: import upstream snapshot with attribution

examples/simple-mcp/README.md

@@ -0,0 +1,156 @@
+# simple-mcp (Simple MCP Provider)
+
+This example demonstrates how to use the MCP provider for evaluating MCP servers. The MCP provider is designed for direct tool calling evaluation rather than text generation, making it ideal for testing tool behavior, security vulnerabilities, and edge cases.
+
+## Quick Start
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example simple-mcp
+cd simple-mcp
+```
+
+## Getting Started
+
+1. Initialize the example:
+
+   ```bash
+   npx promptfoo@latest init --example simple-mcp
+   ```
+
+2. Navigate to the example directory:
+
+   ```bash
+   cd simple-mcp
+   ```
+
+3. Install the example dependencies, including the optional MCP SDK used by `example-server.js`:
+
+   ```bash
+   npm install
+   ```
+
+4. Configure your MCP server in `promptfooconfig.yaml`
+
+5. Run the evaluation:
+   ```bash
+   npx promptfoo eval
+   ```
+
+## Configuration Examples
+
+### Basic Security Testing
+
+```yaml
+providers:
+  - id: mcp
+    config:
+      enabled: true
+      servers:
+        - name: security-test-server
+          path: ./example-server.js
+
+tests:
+  # Test path traversal prevention
+  - vars:
+      prompt: '{"tool": "read_file", "args": {"path": "../../../etc/passwd"}}'
+    assert:
+      - type: contains
+        value: 'Path traversal not allowed'
+
+  # Test command injection prevention
+  - vars:
+      prompt: '{"tool": "execute_command", "args": {"command": "rm -rf /"}}'
+    assert:
+      - type: contains
+        value: 'Dangerous command blocked'
+```
+
+### Advanced Security Testing
+
+Test various security scenarios and edge cases:
+
+```yaml
+tests:
+  # SSRF prevention
+  - vars:
+      prompt: '{"tool": "fetch_url", "args": {"url": "http://localhost:8080/admin"}}'
+    assert:
+      - type: contains
+        value: 'Internal network access blocked'
+
+  # SQL injection prevention
+  - vars:
+      prompt: '{"tool": "query_database", "args": {"query": "SELECT * FROM users; DROP TABLE users;"}}'
+    assert:
+      - type: contains
+        value: 'dangerous SQL query blocked'
+
+  # Data previewing
+  - vars:
+      prompt: '{"tool": "process_data", "args": {"data": "Hello from the MCP example", "operation": "preview"}}'
+    assert:
+      - type: contains
+        value: 'Preview: Hello from the MCP example'
+```
+
+### Debug Mode
+
+Enable debug mode to see detailed information about MCP connections and tool calls:
+
+```yaml
+providers:
+  - id: mcp
+    config:
+      enabled: true
+      debug: true
+      verbose: true
+      servers:
+        - name: my-server
+          url: http://localhost:3000/mcp
+```
+
+### Custom Response Parsing
+
+The example also includes `response-parser.js`, which reads `structuredContent` from the raw MCP
+tool result and falls back to Promptfoo's normalized `content` string:
+
+```yaml
+providers:
+  - id: mcp
+    config:
+      enabled: true
+      servers:
+        - name: security-test-server
+          path: ./example-server.js
+      transformResponse: 'file://response-parser.js'
+```
+
+```javascript
+export default function parseMcpResponse(result, content) {
+  return result.structuredContent?.summary ?? content;
+}
+```
+
+The `get_user_profile` test proves the parser is reading structured MCP output by asserting on
+`Ada Lovelace is active`, which is not present in the tool's text content.
+Function and file-based transforms may also be async when parsing requires additional work.
+
+Relative file reads in `example-server.js` are resolved from the example directory, so the bundled
+tests behave the same whether you run them from the copied example folder or from the promptfoo repo
+root during local development.
+
+## Example MCP Servers
+
+For testing, you can use example MCP servers:
+
+- **Local Node.js Server**: Create a simple MCP server using the `@modelcontextprotocol/sdk`
+- **Python Server**: Use the Python MCP SDK to create custom tools
+- **HTTP Server**: Any HTTP endpoint that implements the MCP protocol
+
+## Next Steps
+
+- Explore the [MCP specification](https://modelcontextprotocol.io) for creating your own servers
+- Check the `redteam-mcp` example for security testing of MCP implementations
+- Combine MCP providers with other providers for comprehensive evaluations

examples/simple-mcp/example-server.js

@@ -0,0 +1,460 @@
+#!/usr/bin/env node
+
+import { exec } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { promisify } from 'util';
+
+/**
+ * Simple MCP Server Example
+ *
+ * This is a basic MCP server that provides text generation tools.
+ * You can run this server locally to test the MCP provider.
+ *
+ * Usage:
+ *   node example-server.js
+ *
+ * Dependencies:
+ *   npm install @modelcontextprotocol/sdk zod
+ */
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+
+// Create MCP server
+const server = new McpServer({
+  name: 'example-text-server',
+  version: '1.0.0',
+});
+
+const execAsync = promisify(exec);
+const serverDir = path.dirname(fileURLToPath(import.meta.url));
+
+// File operations
+function readFile(filePath, encoding = 'utf8') {
+  // Security check - prevent path traversal
+  if (filePath.includes('..') || filePath.includes('~')) {
+    throw new Error('Invalid path: Path traversal not allowed');
+  }
+
+  // Simulate reading restricted files
+  if (filePath.includes('/etc/passwd') || filePath.includes('/etc/shadow')) {
+    throw new Error('Permission denied: Cannot read system files');
+  }
+
+  try {
+    const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(serverDir, filePath);
+    if (encoding === 'base64') {
+      return fs.readFileSync(resolvedPath).toString('base64');
+    }
+    return fs.readFileSync(resolvedPath, encoding);
+  } catch (error) {
+    throw new Error(`File read error: ${error.message}`);
+  }
+}
+
+function writeFile(path, content, mode = 'write') {
+  // Security check - prevent writing to system directories
+  if (path.startsWith('/etc/') || path.startsWith('/sys/') || path.startsWith('/proc/')) {
+    throw new Error('Permission denied: Cannot write to system directories');
+  }
+
+  try {
+    if (mode === 'append') {
+      fs.appendFileSync(path, content);
+    } else {
+      fs.writeFileSync(path, content);
+    }
+    return `Successfully ${mode === 'append' ? 'appended to' : 'wrote'} file: ${path}`;
+  } catch (error) {
+    throw new Error(`File write error: ${error.message}`);
+  }
+}
+
+// Command execution
+async function executeCommand(command, args = [], timeout = 5000) {
+  // Security check - block dangerous commands
+  const blockedCommands = ['rm', 'del', 'format', 'sudo', 'su', 'chmod', 'chown'];
+  if (blockedCommands.some((cmd) => command.toLowerCase().includes(cmd))) {
+    throw new Error('Security violation: Dangerous command blocked');
+  }
+
+  try {
+    const fullCommand = `${command} ${args.join(' ')}`;
+    const { stdout, stderr } = await execAsync(fullCommand, { timeout });
+    return stdout || stderr;
+  } catch (error) {
+    if (error.killed) {
+      throw new Error(`Command timed out after ${timeout}ms`);
+    }
+    throw new Error(`Command execution failed: ${error.message}`);
+  }
+}
+
+// URL fetching
+async function fetchUrl(url, method = 'GET', headers = {}, body = null) {
+  // Security check - prevent SSRF to internal networks
+  if (url.includes('localhost') || url.includes('127.0.0.1') || url.includes('192.168.')) {
+    throw new Error('Security violation: Internal network access blocked');
+  }
+
+  try {
+    // Simulate HTTP request
+    return `Mock response from ${method} ${url} with headers: ${JSON.stringify(headers)}`;
+  } catch (error) {
+    throw new Error(`HTTP request failed: ${error.message}`);
+  }
+}
+
+// Database operations
+function queryDatabase(query, database = 'default', params = []) {
+  // Security check - prevent SQL injection patterns
+  const dangerousPatterns = ['DROP', 'DELETE', 'UPDATE', 'INSERT', '--', ';'];
+  if (dangerousPatterns.some((pattern) => query.toUpperCase().includes(pattern))) {
+    throw new Error('Security violation: Potentially dangerous SQL query blocked');
+  }
+
+  // Simulate database query
+  return `Mock query result for: ${query} on database: ${database} with params: ${JSON.stringify(params)}`;
+}
+
+// Data processing
+function processData(data, operation, format = 'text') {
+  try {
+    switch (operation) {
+      case 'validate':
+        if (format === 'json') {
+          JSON.parse(data);
+          return 'Valid JSON data';
+        }
+        return 'Data validation completed';
+
+      case 'transform':
+        return `Transformed data: ${data.substring(0, 50)}...`;
+
+      case 'extract':
+        return `Extracted fields from ${format} data`;
+
+      case 'preview':
+        return `Preview: ${data.substring(0, 50)}`;
+
+      default:
+        throw new Error(`Unknown operation: ${operation}`);
+    }
+  } catch (error) {
+    throw new Error(`Data processing failed: ${error.message}`);
+  }
+}
+
+// System information
+function getSystemInfo(infoType, detailed = false) {
+  const info = {
+    cpu: detailed ? 'CPU: Intel i7-9700K @ 3.60GHz, 8 cores' : 'CPU: Intel i7',
+    memory: detailed ? 'Memory: 16GB DDR4, 8GB available' : 'Memory: 16GB',
+    disk: detailed ? 'Disk: 500GB SSD, 200GB free' : 'Disk: 500GB',
+    network: detailed ? 'Network: Ethernet connected, WiFi available' : 'Network: Connected',
+    processes: detailed ? 'Processes: 156 running, top: chrome (15%), node (8%)' : 'Processes: 156',
+    environment: detailed ? 'Environment: Production, Node.js v20.0.0' : 'Environment: Production',
+  };
+
+  return info[infoType] || 'Unknown system information type';
+}
+
+// Register tools with the MCP server
+server.registerTool(
+  'read_file',
+  {
+    title: 'Read File',
+    description: 'Read contents of a file from the local filesystem',
+    inputSchema: {
+      path: z.string().describe('File path to read'),
+      encoding: z.enum(['utf8', 'base64', 'binary']).prefault('utf8').describe('File encoding'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = readFile(args.path, args.encoding);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: String(result),
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'write_file',
+  {
+    title: 'Write File',
+    description: 'Write content to a file on the local filesystem',
+    inputSchema: {
+      path: z.string().describe('File path to write to'),
+      content: z.string().describe('Content to write to the file'),
+      mode: z.enum(['write', 'append']).prefault('write').describe('Write mode'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = writeFile(args.path, args.content, args.mode);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: result,
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'execute_command',
+  {
+    title: 'Execute Command',
+    description: 'Execute a system command',
+    inputSchema: {
+      command: z.string().describe('Command to execute'),
+      args: z.array(z.string()).prefault([]).describe('Command arguments'),
+      timeout: z.number().prefault(5000).describe('Timeout in milliseconds'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = await executeCommand(args.command, args.args, args.timeout);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: String(result),
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'fetch_url',
+  {
+    title: 'Fetch URL',
+    description: 'Fetch content from a URL',
+    inputSchema: {
+      url: z.string().describe('URL to fetch'),
+      method: z.enum(['GET', 'POST', 'PUT', 'DELETE']).prefault('GET').describe('HTTP method'),
+      headers: z.record(z.string(), z.string()).prefault({}).describe('HTTP headers'),
+      body: z.string().optional().describe('Request body for POST/PUT requests'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = await fetchUrl(args.url, args.method, args.headers, args.body);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: result,
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'query_database',
+  {
+    title: 'Query Database',
+    description: 'Execute a database query',
+    inputSchema: {
+      query: z.string().describe('SQL query to execute'),
+      database: z.string().prefault('default').describe('Database name'),
+      params: z.array(z.string()).prefault([]).describe('Query parameters'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = queryDatabase(args.query, args.database, args.params);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: result,
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'process_data',
+  {
+    title: 'Process Data',
+    description: 'Process and transform data',
+    inputSchema: {
+      data: z.string().describe('Data to process (JSON string or plain text)'),
+      operation: z
+        .enum(['validate', 'transform', 'extract', 'preview'])
+        .describe('Operation to perform'),
+      format: z
+        .enum(['json', 'xml', 'csv', 'text'])
+        .prefault('text')
+        .describe('Expected data format'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = processData(args.data, args.operation, args.format);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: result,
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'get_system_info',
+  {
+    title: 'Get System Info',
+    description: 'Get system information',
+    inputSchema: {
+      info_type: z
+        .enum(['cpu', 'memory', 'disk', 'network', 'processes', 'environment'])
+        .describe('Type of system information'),
+      detailed: z.boolean().prefault(false).describe('Return detailed information'),
+    },
+  },
+  async (args) => {
+    try {
+      const result = getSystemInfo(args.info_type, args.detailed);
+      return {
+        content: [
+          {
+            type: 'text',
+            text: result,
+          },
+        ],
+      };
+    } catch (error) {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `Error: ${error.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  },
+);
+
+server.registerTool(
+  'get_user_profile',
+  {
+    title: 'Get User Profile',
+    description: 'Return a structured user profile for parser demonstrations',
+    inputSchema: {
+      user_id: z.string().describe('User identifier'),
+    },
+  },
+  async (args) => ({
+    content: [
+      {
+        type: 'text',
+        text: `Profile lookup complete for ${args.user_id}`,
+      },
+    ],
+    structuredContent: {
+      user: {
+        id: args.user_id,
+        name: 'Ada Lovelace',
+        status: 'active',
+      },
+      summary: 'Ada Lovelace is active',
+    },
+  }),
+);
+
+// Start the server
+async function main() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('Example MCP Server running...');
+}
+
+main().catch((error) => {
+  console.error('Server error:', error);
+  process.exit(1);
+});

examples/simple-mcp/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "simple-mcp-example",
+  "version": "1.0.0",
+  "description": "Example MCP server for testing the promptfoo MCP provider",
+  "type": "module",
+  "main": "example-server.js",
+  "scripts": {
+    "start": "node example-server.js",
+    "test": "promptfoo eval"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "promptfoo": "*"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "promptfoo",
+    "security-testing",
+    "red-team"
+  ],
+  "author": "promptfoo",
+  "license": "MIT"
+}

examples/simple-mcp/promptfooconfig.yaml

@@ -0,0 +1,126 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'MCP tool evaluation and security testing'
+
+# Note: The MCP provider expects JSON-formatted tool calls in the 'prompt' variable.
+# Format: {"tool": "tool_name", "args": {"param1": "value1", "param2": "value2"}}
+# Alternative field names are also supported (see README.md for details)
+
+prompts:
+  - '{{prompt}}'
+
+providers:
+  - id: mcp
+    label: 'MCP Security Tester'
+    config:
+      enabled: true
+      servers:
+        - name: security-test-server
+          path: ./example-server.js
+      transformResponse: 'file://response-parser.js'
+      verbose: true
+      debug: true
+
+tests:
+  # Test basic file reading
+  - vars:
+      prompt: '{"tool": "read_file", "args": {"path": "package.json"}}'
+    assert:
+      - type: javascript
+        value: 'output && output.trim().length > 0'
+      - type: contains
+        value: 'simple-mcp-example'
+
+  # Test path traversal prevention
+  - vars:
+      prompt: '{"tool": "read_file", "args": {"path": "../../../etc/passwd"}}'
+    assert:
+      - type: contains
+        value: 'Path traversal not allowed'
+
+  # Test restricted file access
+  - vars:
+      prompt: '{"tool": "read_file", "args": {"path": "/etc/passwd"}}'
+    assert:
+      - type: contains
+        value: 'Permission denied'
+
+  # Test command execution blocking
+  - vars:
+      prompt: '{"tool": "execute_command", "args": {"command": "rm -rf /"}}'
+    assert:
+      - type: contains
+        value: 'Dangerous command blocked'
+
+  # Test safe command execution
+  - vars:
+      prompt: '{"tool": "execute_command", "args": {"command": "echo", "args": ["Hello World"]}}'
+    assert:
+      - type: contains
+        value: 'Hello World'
+
+  # Test SSRF prevention
+  - vars:
+      prompt: '{"tool": "fetch_url", "args": {"url": "http://localhost:8080/admin"}}'
+    assert:
+      - type: contains
+        value: 'Internal network access blocked'
+
+  # Test SQL injection prevention
+  - vars:
+      prompt: '{"tool": "query_database", "args": {"query": "SELECT * FROM users; DROP TABLE users;"}}'
+    assert:
+      - type: contains
+        value: 'dangerous SQL query blocked'
+
+  # Test safe database query
+  - vars:
+      prompt: '{"tool": "query_database", "args": {"query": "SELECT name FROM products WHERE category = ?", "params": ["electronics"]}}'
+    assert:
+      - type: contains
+        value: 'Mock query result'
+
+  # Test data previewing
+  - vars:
+      prompt: '{"tool": "process_data", "args": {"data": "Hello World from the MCP example", "operation": "preview"}}'
+    assert:
+      - type: contains
+        value: 'Preview: Hello World from the MCP example'
+
+  # Test JSON validation
+  - vars:
+      prompt: '{"tool": "process_data", "args": {"data": "{\"valid\": \"json\"}", "operation": "validate", "format": "json"}}'
+    assert:
+      - type: contains
+        value: 'Valid JSON data'
+
+  # Test system information access
+  - vars:
+      prompt: '{"tool": "get_system_info", "args": {"info_type": "cpu", "detailed": true}}'
+    assert:
+      - type: contains
+        value: 'Intel i7'
+
+  # Test custom response parsing from structured MCP content
+  - vars:
+      prompt: '{"tool": "get_user_profile", "args": {"user_id": "user-123"}}'
+    assert:
+      - type: equals
+        value: 'Ada Lovelace is active'
+
+  # Test file write restrictions
+  - vars:
+      prompt: '{"tool": "write_file", "args": {"path": "/etc/shadow", "content": "malicious content"}}'
+    assert:
+      - type: contains
+        value: 'Permission denied'
+
+  # Test with alternative field names (using "function" instead of "tool")
+  - vars:
+      prompt: '{"function": "get_system_info", "arguments": {"info_type": "memory", "detailed": false}}'
+    assert:
+      - type: contains
+        value: 'Memory'
+
+evaluateOptions:
+  maxConcurrency: 1 # MCP connections may need to be sequential
+  delay: 500 # Short delay between tool calls

examples/simple-mcp/response-parser.js

@@ -0,0 +1,3 @@
+export default function parseMcpResponse(result, content) {
+  return result.structuredContent?.summary ?? content;
+}