chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,64 @@
+# openai-codex-app-server (OpenAI Codex App Server Examples)
+
+These examples evaluate Codex through the experimental `codex app-server` protocol.
+
+## Setup
+
+Install and sign in to Codex:
+
+```bash
+npm i -g @openai/codex
+codex
+```
+
+You can also use an API key:
+
+```bash
+export OPENAI_API_KEY=your_api_key_here
+```
+
+## Run
+
+From this repository:
+
+```bash
+npm run local -- eval -c examples/openai-codex-app-server/promptfooconfig.yaml --no-cache
+```
+
+With an `.env` file:
+
+```bash
+npm run local -- eval -c examples/openai-codex-app-server/promptfooconfig.yaml --env-file .env --no-cache
+```
+
+If you initialized this example separately:
+
+```bash
+npx promptfoo@latest init --example openai-codex-app-server
+cd openai-codex-app-server
+npx promptfoo@latest eval -c promptfooconfig.yaml --no-cache
+```
+
+## Included Configs
+
+- `promptfooconfig.yaml` - Read-only structured repo summary.
+- `promptfooconfig.tracing.yaml` - Enables OTEL tracing and asserts on the `gen_ai.turn *` protocol-turn marker spans (see [Turn marker spans](https://www.promptfoo.dev/docs/tracing/#per-llm-turn-spans)).
+- `approval-policy/promptfooconfig.yaml` - Demonstrates deterministic approval request handling.
+- `review-diff/promptfooconfig.yaml` - Uses Codex app-server to review the current git diff.
+- `skills/promptfooconfig.yaml` - Demonstrates explicit skill input items.
+
+The skills config expects `CODEX_SKILL_CREATOR_PATH` to point at a local
+`skill-creator/SKILL.md` file.
+
+## Notes
+
+The provider starts its own `codex app-server` process. It does not attach to an already-running Codex Desktop app process.
+
+The default examples use:
+
+- `sandbox_mode: read-only`
+- `approval_policy: never`
+- `skip_git_repo_check: true`
+- `thread_cleanup: unsubscribe`
+
+See [OpenAI Codex App Server Provider Documentation](/docs/providers/openai-codex-app-server/) for full configuration details.
@@ -0,0 +1,41 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Codex app-server approval handling
+
+prompts:
+  - |
+    Try to run `touch approval-check.txt` with a shell command.
+    If the sandbox blocks it, request the permission needed to rerun it.
+    Explain whether the command was allowed.
+
+providers:
+  - id: openai:codex-app-server:gpt-5.5
+    config:
+      sandbox_mode: read-only
+      approval_policy: on-request
+      approvals_reviewer: user
+      skip_git_repo_check: true
+      server_request_policy:
+        command_execution: decline
+        file_change: decline
+        mcp_elicitation: decline
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: |
+        const requests = context.providerResponse?.metadata?.codexAppServer?.serverRequests ?? [];
+        const commandRequest = requests.find((request) =>
+          String(request.method).includes('commandExecution') ||
+          String(request.method).includes('execCommandApproval')
+        );
+
+        return {
+          pass: Boolean(commandRequest),
+          score: commandRequest ? 1 : 0,
+          reason: commandRequest
+            ? 'Observed a deterministic command approval request.'
+            : 'No command approval request was observed.'
+        };
+
+tests:
+  - vars: {}
@@ -0,0 +1,32 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Codex app-server: verify traced protocol turn markers'
+
+prompts:
+  - 'Return a concise JSON summary of this workspace. Include the main purpose.'
+
+providers:
+  - id: openai:codex-app-server:gpt-5.5
+    config:
+      sandbox_mode: read-only
+      approval_policy: never
+      skip_git_repo_check: true
+
+tracing:
+  enabled: true
+  otlp:
+    http:
+      enabled: true
+      port: 4318
+      host: '127.0.0.1'
+      acceptFormats: ['json']
+
+tests:
+  - vars: {}
+    metadata:
+      tracingEnabled: true
+    assert:
+      - type: is-json
+      - type: trace-span-count
+        value:
+          pattern: 'gen_ai.turn *'
+          min: 1
@@ -0,0 +1,34 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Codex app-server structured repo review
+
+prompts:
+  - 'Return a concise JSON summary of this workspace. Include the main purpose and two likely maintenance risks.'
+
+providers:
+  - id: openai:codex-app-server:gpt-5.5
+    config:
+      sandbox_mode: read-only
+      approval_policy: never
+      skip_git_repo_check: true
+      output_schema:
+        type: object
+        properties:
+          summary:
+            type: string
+          risks:
+            type: array
+            items:
+              type: string
+        required: [summary, risks]
+        additionalProperties: false
+
+defaultTest:
+  assert:
+    - type: is-json
+    - type: javascript
+      value: |
+        const parsed = JSON.parse(output);
+        return typeof parsed.summary === 'string' && Array.isArray(parsed.risks);
+
+tests:
+  - vars: {}
@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Codex app-server git diff review
+
+prompts:
+  - |
+    Review the current git diff in this workspace.
+
+    Focus only on actionable correctness, security, data-loss, resource-leak,
+    API-contract, and missing-test risks introduced by the diff. Ignore style-only
+    issues.
+
+    Return JSON with:
+    - comments: an array of review comments
+    - summary: one short sentence
+
+    Each comment must include file, line, severity, and body. If there are no
+    actionable comments, return an empty comments array.
+
+providers:
+  - id: openai:codex-app-server:gpt-5.5
+    config:
+      sandbox_mode: read-only
+      approval_policy: never
+      skip_git_repo_check: false
+      turn_timeout_ms: 900000
+      output_schema:
+        type: object
+        properties:
+          comments:
+            type: array
+            items:
+              type: object
+              properties:
+                file:
+                  type: string
+                line:
+                  type: integer
+                severity:
+                  type: string
+                body:
+                  type: string
+              required: [file, line, severity, body]
+              additionalProperties: false
+          summary:
+            type: string
+        required: [comments, summary]
+        additionalProperties: false
+
+defaultTest:
+  assert:
+    - type: is-json
+
+tests:
+  - vars: {}
@@ -0,0 +1,23 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Codex app-server skill input
+
+prompts:
+  - file://skill-input.json
+
+providers:
+  - id: openai:codex-app-server:gpt-5.5
+    config:
+      sandbox_mode: read-only
+      approval_policy: never
+      skip_git_repo_check: true
+
+defaultTest:
+  assert:
+    - type: contains-any
+      value:
+        - test
+        - verify
+        - checklist
+
+tests:
+  - vars: {}
@@ -0,0 +1,11 @@
+[
+  {
+    "type": "text",
+    "text": "$skill-creator Draft a checklist for verifying a new promptfoo provider."
+  },
+  {
+    "type": "skill",
+    "name": "skill-creator",
+    "path": "{{env.CODEX_SKILL_CREATOR_PATH}}"
+  }
+]