chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,53 @@
+# provider-github-models (GitHub Models Provider)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example provider-github-models
+cd provider-github-models
+```
+
+## Setup
+
+Set your `GITHUB_TOKEN` environment variable. You can create a Personal Access Token at https://github.com/settings/tokens.
+
+```bash
+export GITHUB_TOKEN=ghp_your_token_here
+```
+
+## What This Tests
+
+This example tests the GitHub Models API integration:
+
+1. **Direct Provider Usage**: Uses GitHub Models as the main provider (gpt-4o-mini and gpt-5-mini)
+2. **GitHub as Grader**: Uses GitHub Models for `llm-rubric` assertions via the `defaultTest.options.provider.text` configuration
+
+## Run the Evaluation
+
+```bash
+promptfoo eval
+```
+
+## View Results
+
+```bash
+promptfoo view
+```
+
+## Available Models
+
+GitHub Models supports various models including:
+
+**OpenAI Models:**
+
+- `github:openai/gpt-4o`, `github:openai/gpt-4o-mini`
+- `github:openai/gpt-5`, `github:openai/gpt-5-mini`, `github:openai/gpt-5-nano`
+- `github:openai/o4-mini`, `github:openai/o3-mini`
+
+**Other Providers:**
+
+- `github:meta/llama-4-scout-17b-16e-instruct`
+- `github:deepseek/DeepSeek-V3-0324`
+- `github:mistral-ai/mistral-large`
+
+See the [GitHub Models marketplace](https://github.com/marketplace?type=models) for the full list.
@@ -0,0 +1,44 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'GitHub Models Provider Test'
+
+prompts:
+  - 'What is the capital of {{country}}?'
+
+providers:
+  - id: github:openai/gpt-4o-mini
+    config:
+      temperature: 0
+
+  - id: github:openai/gpt-5-mini
+    config:
+      temperature: 0
+
+defaultTest:
+  options:
+    provider:
+      text: github:openai/gpt-4o-mini
+
+tests:
+  - vars:
+      country: France
+    assert:
+      - type: contains
+        value: Paris
+      - type: llm-rubric
+        value: 'The response correctly identifies Paris as the capital of France'
+
+  - vars:
+      country: Japan
+    assert:
+      - type: contains
+        value: Tokyo
+      - type: llm-rubric
+        value: 'The response correctly identifies Tokyo as the capital of Japan'
+
+  - vars:
+      country: Australia
+    assert:
+      - type: contains
+        value: Canberra
+      - type: llm-rubric
+        value: 'The response correctly identifies Canberra as the capital of Australia'