chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,188 @@
+# provider-truefoundry (TrueFoundry Examples)
+
+This directory contains example configurations for using [TrueFoundry's AI Gateway](https://www.truefoundry.com/ai-gateway) (LLM, MCP, and Agent Gateway) with promptfoo.
+
+## Prerequisites
+
+1. **TrueFoundry API Key**: Obtain your API key from the [TrueFoundry Console](https://www.truefoundry.com/)
+
+2. **Set Environment Variable**:
+
+   ```bash
+   export TRUEFOUNDRY_API_KEY=your_api_key_here
+   ```
+
+## Quick Start
+
+To quickly set up this example:
+
+```bash
+npx promptfoo@latest init --example provider-truefoundry
+cd provider-truefoundry
+```
+
+## Examples
+
+### 1. Basic Configuration (`promptfooconfig.yaml`)
+
+A simple example demonstrating basic TrueFoundry usage with multiple models:
+
+- GPT-4 via OpenAI
+- Claude 3.5 Sonnet via Anthropic
+- Custom metadata and logging configuration
+
+**Run the example:**
+
+```bash
+npx promptfoo eval -c promptfooconfig.yaml
+```
+
+**View results:**
+
+```bash
+npx promptfoo view
+```
+
+### 2. MCP Servers (`promptfooconfig-mcp.yaml`)
+
+An advanced example showcasing TrueFoundry's MCP (Model Context Protocol) server integration:
+
+- Web search capabilities
+- Code execution tools
+- Multi-step reasoning with tool use
+- Custom iteration limits
+
+**Run the example:**
+
+```bash
+npx promptfoo eval -c promptfooconfig-mcp.yaml
+```
+
+## Configuration Options
+
+### Basic Options
+
+```yaml
+providers:
+  - id: truefoundry:openai-main/gpt-5
+    config:
+      temperature: 0.7
+      max_completion_tokens: 500
+```
+
+**Note**: The model identifier format is `provider-account/model-name`. The `provider-account` is the name of your provider integration in TrueFoundry (e.g., `openai-main`, `anthropic-main`).
+
+### Metadata and Logging
+
+```yaml
+config:
+  metadata:
+    user_id: 'your-user-id'
+    environment: 'development'
+    custom_key: 'custom_value'
+  loggingConfig:
+    enabled: true
+```
+
+### MCP Servers
+
+```yaml
+config:
+  mcp_servers:
+    - integration_fqn: 'common-tools'
+      enable_all_tools: false
+      tools:
+        - name: 'web_search'
+        - name: 'code_executor'
+  iteration_limit: 20
+```
+
+## Supported Models
+
+TrueFoundry provides access to models from multiple providers. Use the format `provider-account/model-name`:
+
+### OpenAI
+
+```yaml
+- truefoundry:openai-main/gpt-5
+- truefoundry:openai-main/gpt-4o
+- truefoundry:openai-main/gpt-4o-mini
+- truefoundry:openai-main/o1
+- truefoundry:openai-main/o1-mini
+```
+
+### Anthropic
+
+```yaml
+- truefoundry:anthropic-main/claude-sonnet-4.5
+- truefoundry:anthropic-main/claude-3-5-sonnet-20241022
+- truefoundry:anthropic-main/claude-3-opus-20240229
+```
+
+### Google Gemini
+
+```yaml
+- truefoundry:vertex-ai-main/gemini-2.5-pro
+- truefoundry:vertex-ai-main/gemini-2.5-flash
+- truefoundry:vertex-ai-main/gemini-1.5-pro
+```
+
+### Other Providers
+
+```yaml
+- truefoundry:groq-main/llama-3.3-70b-versatile
+- truefoundry:mistral-main/mistral-large-latest
+```
+
+## Embeddings
+
+TrueFoundry also supports embedding models:
+
+```yaml
+providers:
+  - id: truefoundry:openai-main/text-embedding-3-large
+    config:
+      metadata:
+        user_id: 'embedding-user'
+      loggingConfig:
+        enabled: true
+```
+
+### Cohere Embeddings
+
+When using Cohere models, specify the `input_type`:
+
+```yaml
+providers:
+  - id: truefoundry:cohere-main/embed-english-v3.0
+    config:
+      input_type: 'search_query' # Options: search_query, search_document, classification, clustering
+```
+
+## Observability
+
+When `loggingConfig.enabled` is set to `true`, all requests are logged in the TrueFoundry dashboard where you can:
+
+- Monitor request/response logs
+- Track performance metrics
+- Analyze costs
+- Debug issues
+- Filter by custom metadata
+
+## Resources
+
+- [TrueFoundry Documentation](https://docs.truefoundry.com/docs/ai-gateway)
+- [TrueFoundry Provider Documentation](/docs/providers/truefoundry/)
+- [Promptfoo Documentation](https://www.promptfoo.dev/docs/)
+
+## Support
+
+For TrueFoundry-specific questions:
+
+- Visit [docs.truefoundry.com](https://docs.truefoundry.com/)
+- Contact TrueFoundry support
+
+For promptfoo-related questions:
+
+- Visit [promptfoo.dev](https://www.promptfoo.dev/)
+- Join the [Discord community](https://discord.gg/promptfoo)
@@ -0,0 +1,63 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+
+description: TrueFoundry with MCP Servers Example
+
+prompts:
+  - |
+    You are a helpful research assistant with access to web search and code execution tools.
+    Answer the following question using the available tools when necessary:
+    {{question}}
+
+providers:
+  - id: truefoundry:openai-main/gpt-5
+    label: 'GPT-5 with MCP Tools'
+    config:
+      temperature: 0.7
+      max_completion_tokens: 1000
+      # MCP Server configuration for enhanced capabilities
+      mcp_servers:
+        - integration_fqn: 'common-tools'
+          enable_all_tools: false
+          tools:
+            - name: 'web_search'
+            - name: 'code_executor'
+      # Maximum iterations for tool calling
+      iteration_limit: 20
+      # Metadata for tracking
+      metadata:
+        user_id: 'mcp-example-user'
+        environment: 'development'
+        use_case: 'research'
+      # Enable logging for observability
+      loggingConfig:
+        enabled: true
+
+tests:
+  - description: 'Web search integration test'
+    vars:
+      question: 'Search for the latest developments in quantum computing and summarize the key findings'
+    assert:
+      - type: llm-rubric
+        value: 'Successfully uses web search to find and summarize recent quantum computing developments'
+      - type: not-contains
+        value: 'I cannot search'
+
+  - description: 'Code execution test'
+    vars:
+      question: 'Calculate the factorial of 10 using code and show your work'
+    assert:
+      - type: llm-rubric
+        value: 'Correctly calculates factorial of 10 (3628800) and shows the calculation'
+
+  - description: 'Multi-step reasoning with tools'
+    vars:
+      question: 'Find the current price of Bitcoin and calculate what 5 BTC would be worth in USD'
+    assert:
+      - type: llm-rubric
+        value: 'Uses web search to find current Bitcoin price and performs correct calculation'
+
+defaultTest:
+  options:
+    provider:
+      config:
+        temperature: 0.7
@@ -0,0 +1,61 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+
+description: TrueFoundry AI Gateway Example
+
+prompts:
+  - |
+    You are a helpful assistant. Answer the following question concisely:
+    {{question}}
+
+providers:
+  # Basic GPT-5 configuration through TrueFoundry
+  - id: truefoundry:openai-main/gpt-5
+    label: 'GPT-5 via TrueFoundry'
+    config:
+      temperature: 0.7
+      max_completion_tokens: 500
+      metadata:
+        user_id: 'example-user'
+        environment: 'development'
+      loggingConfig:
+        enabled: true
+
+  # Claude Sonnet 4.5 for comparison
+  - id: truefoundry:anthropic-main/claude-sonnet-4.5
+    label: 'Claude Sonnet 4.5 via TrueFoundry'
+    config:
+      temperature: 0.7
+      max_tokens: 500
+      metadata:
+        user_id: 'example-user'
+        environment: 'development'
+      loggingConfig:
+        enabled: true
+
+tests:
+  - description: 'Basic knowledge test'
+    vars:
+      question: 'What is the capital of France?'
+    assert:
+      - type: contains
+        value: 'Paris'
+
+  - description: 'Technical explanation'
+    vars:
+      question: 'Explain machine learning in simple terms'
+    assert:
+      - type: llm-rubric
+        value: 'Provides a clear, simple explanation of machine learning'
+
+  - description: 'Creative writing'
+    vars:
+      question: 'Write a haiku about coding'
+    assert:
+      - type: llm-rubric
+        value: 'Responds with a proper haiku (5-7-5 syllable structure)'
+
+defaultTest:
+  options:
+    provider:
+      config:
+        temperature: 0.7