chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,8 @@
+# provider-cloudflare (Cloudflare)
+
+Examples for using promptfoo with Cloudflare AI services.
+
+## Examples
+
+- [ai](./ai/) - Cloudflare Workers AI native models (Llama, GPT-OSS, etc.)
+- [gateway](./gateway/) - Cloudflare AI Gateway proxy for OpenAI, Anthropic, Groq, etc.
@@ -0,0 +1,94 @@
+# provider-cloudflare/ai (Cloudflare Workers AI)
+
+Cloudflare Workers AI evaluation with OpenAI-compatible endpoints.
+
+## Quick Start
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example provider-cloudflare/ai
+cd provider-cloudflare/ai
+```
+
+## Environment Variables
+
+This example requires the following environment variables:
+
+- `CLOUDFLARE_ACCOUNT_ID` - Your Cloudflare account ID (found in your Cloudflare dashboard)
+- `CLOUDFLARE_API_KEY` - Your Cloudflare API key with Workers AI permissions
+
+Set these in your environment:
+
+```bash
+export CLOUDFLARE_ACCOUNT_ID=your_account_id_here
+export CLOUDFLARE_API_KEY=your_api_key_here
+```
+
+## Example Configurations
+
+### Basic Usage (`chat_config.yaml`)
+
+Compares the latest flagship chat models:
+
+- `cloudflare-ai:chat:@cf/openai/gpt-oss-120b` - OpenAI's production, general purpose, high reasoning model
+- `cloudflare-ai:chat:@cf/meta/llama-4-scout-17b-16e-instruct` - Meta's Llama 4 Scout with native multimodal capabilities
+
+### Advanced Configuration (`chat_advanced_configuration.yaml`)
+
+Demonstrates OpenAI-compatible parameters with Cloudflare AI:
+
+- Uses `@cf/mistralai/mistral-small-3.1-24b-instruct` (enhanced vision understanding and 128K context)
+- Custom `max_tokens`, `temperature`, `seed` parameters
+- Environment variable configuration patterns
+
+### Embedding Configuration (`embedding_configuration.yaml`)
+
+Shows embedding generation and similarity testing:
+
+- Chat: `@cf/meta/llama-3.3-70b-instruct-fp8-fast` (optimized 70B model with fp8 quantization)
+- Embedding: `@cf/google/embeddinggemma-300m` (state-of-the-art embedding model trained on 100+ languages)
+- Similarity assertion testing with configurable thresholds
+
+## Provider Types
+
+Cloudflare AI supports three provider types:
+
+1. **Chat**: `cloudflare-ai:chat:model-name` - Conversational AI and instruction following
+2. **Completion**: `cloudflare-ai:completion:model-name` - Text completion and generation
+3. **Embedding**: `cloudflare-ai:embedding:model-name` - Text embeddings for similarity and search
+
+## Featured Models (2025)
+
+This example showcases the latest flagship models:
+
+- **OpenAI GPT-OSS-120B** (`@cf/openai/gpt-oss-120b`) - Production-ready, high reasoning capabilities
+- **Meta Llama 4 Scout** (`@cf/meta/llama-4-scout-17b-16e-instruct`) - Multimodal with mixture-of-experts
+- **Meta Llama 3.3 70B Fast** (`@cf/meta/llama-3.3-70b-instruct-fp8-fast`) - Speed-optimized 70B model
+- **Mistral Small 3.1** (`@cf/mistralai/mistral-small-3.1-24b-instruct`) - Enhanced vision and 128K context
+- **EmbeddingGemma-300M** (`@cf/google/embeddinggemma-300m`) - Multilingual embeddings (100+ languages)
+
+## OpenAI Compatibility
+
+All examples use Cloudflare's OpenAI-compatible endpoints, supporting standard parameters:
+
+- `temperature` - Response randomness control
+- `max_tokens` - Output length limits
+- `top_p` - Nucleus sampling
+- `frequency_penalty` - Repetition reduction
+- `presence_penalty` - Topic diversity
+
+## Local Development
+
+For local testing with your development version:
+
+```bash
+npm run local -- eval -c examples/provider-cloudflare/ai/chat_config.yaml
+```
+
+## References
+
+- [Cloudflare Workers AI Models](https://developers.cloudflare.com/workers-ai/models/) - Complete model catalog
+- [Cloudflare Workers AI Documentation](https://developers.cloudflare.com/workers-ai/) - Platform overview
+- [OpenAI Compatibility](https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/) - API details
+- [promptfoo Cloudflare AI Provider](../../../site/docs/providers/cloudflare-ai.md) - Provider documentation
@@ -0,0 +1,25 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+prompts:
+  - Tell me a funny joke about {{topic}}
+
+providers:
+  - id: cloudflare-ai:chat:@cf/mistralai/mistral-small-3.1-24b-instruct
+    config:
+      accountId: your_account_id_here
+      # ===============
+      # API key is loaded from CLOUDFLARE_API_KEY environment variable
+      # Alternatively, use custom environment variable names:
+      # apiKeyEnvar: CUSTOM_CLOUDFLARE_KEY
+      # accountIdEnvar: CUSTOM_CLOUDFLARE_ACCOUNT
+      # ===============
+      # OpenAI-compatible model parameters
+      max_tokens: 800
+      temperature: 0.7
+      seed: 1
+
+tests:
+  - vars:
+      topic: programming
+    assert:
+      - type: icontains
+        value: '{{topic}}'
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+prompts:
+  - What is the capital of the {{country}}?
+
+providers:
+  - cloudflare-ai:chat:@cf/openai/gpt-oss-120b
+  - cloudflare-ai:chat:@cf/meta/llama-4-scout-17b-16e-instruct
+
+tests:
+  - vars:
+      country: United States
+    assert:
+      - type: icontains
+        value: Washington, D.C.
+  - vars:
+      country: England
+    assert:
+      - type: icontains
+        value: London
@@ -0,0 +1,31 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+prompts:
+  - This prompt will be evaluated for embedding similarity
+
+defaultTest:
+  options:
+    provider:
+      embedding:
+        id: cloudflare-ai:embedding:@cf/google/embeddinggemma-300m
+        config:
+          accountId: your_account_id_here
+
+providers:
+  - id: cloudflare-ai:chat:@cf/meta/llama-3.3-70b-instruct-fp8-fast
+    config:
+      accountId: your_account_id_here
+      # ===============
+      # API key is loaded from CLOUDFLARE_API_KEY environment variable
+      # Alternatively, use custom environment variable names:
+      # apiKeyEnvar: CUSTOM_CLOUDFLARE_KEY
+      # accountIdEnvar: CUSTOM_CLOUDFLARE_ACCOUNT
+      # ===============
+      # OpenAI-compatible model parameters
+      max_tokens: 800
+      temperature: 0.6
+
+tests:
+  - assert:
+      - type: similar
+        value: embedding similarity
+        threshold: 0.6
@@ -0,0 +1,55 @@
+# provider-cloudflare/gateway (Cloudflare AI Gateway)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example provider-cloudflare/gateway
+cd provider-cloudflare/gateway
+```
+
+This example demonstrates how to route AI requests through [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/) for caching, rate limiting, and analytics.
+
+## Prerequisites
+
+1. A Cloudflare account with AI Gateway enabled
+2. An AI Gateway created in your Cloudflare dashboard
+3. API keys for the providers you want to use (OpenAI, Anthropic, Groq, etc.)
+
+## Setup
+
+Set the required environment variables:
+
+```bash
+export CLOUDFLARE_ACCOUNT_ID=your_account_id
+export CLOUDFLARE_GATEWAY_ID=your_gateway_id
+export OPENAI_API_KEY=your_openai_key
+export ANTHROPIC_API_KEY=your_anthropic_key
+export GROQ_API_KEY=your_groq_key
+```
+
+## Running the Example
+
+```bash
+npx promptfoo eval -c examples/provider-cloudflare/gateway/promptfooconfig.yaml
+```
+
+## What This Example Does
+
+The configuration compares responses from three different providers, all routed through Cloudflare AI Gateway:
+
+- **OpenAI** (gpt-4o-mini)
+- **Anthropic** (claude-haiku-4-5-20251001)
+- **Groq** (llama-3.1-8b-instant)
+
+Each provider answers questions about technical topics, and the assertions verify that responses contain relevant keywords.
+
+## Benefits of Using AI Gateway
+
+- **Caching**: Identical requests are cached, reducing costs during development
+- **Analytics**: View usage across all providers in your Cloudflare dashboard
+- **Rate Limiting**: Protect against quota issues with request queuing
+- **Logging**: All requests and responses are logged for debugging
+
+## Documentation
+
+See the [Cloudflare AI Gateway provider documentation](https://www.promptfoo.dev/docs/providers/cloudflare-gateway) for more details.
@@ -0,0 +1,89 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+# Cloudflare AI Gateway Example
+#
+# This example demonstrates routing AI requests through Cloudflare AI Gateway
+# for caching, rate limiting, and analytics.
+#
+# Required environment variables:
+#   CLOUDFLARE_ACCOUNT_ID - Your Cloudflare account ID
+#   CLOUDFLARE_GATEWAY_ID - Your AI Gateway ID
+#
+# Optional environment variables:
+#   CF_AIG_TOKEN - Required if your gateway has "Authenticated Gateway" enabled
+#
+# Provider API keys (choose based on which providers you use):
+#   OPENAI_API_KEY - OpenAI API key
+#   ANTHROPIC_API_KEY - Anthropic API key
+#   GROQ_API_KEY - Groq API key
+#
+# Note: If you've configured BYOK (Bring Your Own Keys) in Cloudflare dashboard,
+# you can omit the provider API keys - Cloudflare will use the stored keys.
+#
+# Run with:
+#   npx promptfoo eval -c examples/provider-cloudflare/gateway/promptfooconfig.yaml
+
+description: Cloudflare AI Gateway comparison
+
+prompts:
+  - 'Explain {{topic}} in simple terms, in about 2-3 sentences.'
+
+providers:
+  # Route OpenAI requests through Cloudflare AI Gateway
+  - id: cloudflare-gateway:openai:gpt-4o-mini
+    config:
+      accountId: '{{env.CLOUDFLARE_ACCOUNT_ID}}'
+      gatewayId: '{{env.CLOUDFLARE_GATEWAY_ID}}'
+      # Uncomment if using authenticated gateway:
+      # cfAigToken: '{{env.CF_AIG_TOKEN}}'
+      temperature: 0.7
+      max_tokens: 200
+
+  # Route Anthropic requests through Cloudflare AI Gateway
+  - id: cloudflare-gateway:anthropic:claude-haiku-4-5-20251001
+    config:
+      accountId: '{{env.CLOUDFLARE_ACCOUNT_ID}}'
+      gatewayId: '{{env.CLOUDFLARE_GATEWAY_ID}}'
+      # Uncomment if using authenticated gateway:
+      # cfAigToken: '{{env.CF_AIG_TOKEN}}'
+      max_tokens: 200
+
+  # Route Groq requests through Cloudflare AI Gateway
+  - id: cloudflare-gateway:groq:llama-3.1-8b-instant
+    config:
+      accountId: '{{env.CLOUDFLARE_ACCOUNT_ID}}'
+      gatewayId: '{{env.CLOUDFLARE_GATEWAY_ID}}'
+      # Uncomment if using authenticated gateway:
+      # cfAigToken: '{{env.CF_AIG_TOKEN}}'
+      temperature: 0.7
+      max_tokens: 200
+
+tests:
+  - vars:
+      topic: machine learning
+    assert:
+      - type: contains-any
+        value:
+          - algorithm
+          - data
+          - learn
+          - pattern
+
+  - vars:
+      topic: quantum computing
+    assert:
+      - type: contains-any
+        value:
+          - qubit
+          - quantum
+          - superposition
+          - computer
+
+  - vars:
+      topic: blockchain
+    assert:
+      - type: contains-any
+        value:
+          - block
+          - chain
+          - ledger
+          - decentralized