chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,34 @@
+# openai-codex-sdk/skill-comparison (Codex Skill Comparison)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example openai-codex-sdk/skill-comparison
+cd openai-codex-sdk/skill-comparison
+```
+
+## Overview
+
+This example compares two versions of the same Codex skill against identical review tasks.
+
+- `fixtures/v1` contains a narrower `review-standards` skill that only calls out weak password hashing.
+- `fixtures/v2` contains a stronger version that also checks timing-safe secret comparison.
+- Both providers share an `output_schema` (declared once via a YAML anchor) so each response is guaranteed to match the review JSON shape.
+- The eval verifies `skill-used`, scores issue recall and precision, and uses `max-score` to select the best output for each test case.
+
+Run it from this directory with:
+
+```bash
+promptfoo eval --no-cache
+```
+
+If you run it from another directory, set these environment variables first:
+
+```bash
+export CODEX_SKILL_COMPARE_V1_DIR=/absolute/path/to/fixtures/v1
+export CODEX_SKILL_COMPARE_V2_DIR=/absolute/path/to/fixtures/v2
+```
+
+The checked-in `sample-codex-home` directory is intentionally empty of auth state. Use an API key, or set `CODEX_HOME_OVERRIDE="$HOME/.codex"` to reuse a local Codex login.
+
+Because this example uses `max-score`, the weaker candidate is expected to fail when Promptfoo marks the stronger output as the winner for a test case.
@@ -0,0 +1,10 @@
+---
+name: review-standards
+description: Use this skill when asked to review authentication code for security issues.
+---
+
+When reviewing authentication code:
+
+1. Check password hashing.
+2. Use the issue id `weak-password-hash` when passwords use SHA-1 or MD5.
+3. Return no more than one issue.
@@ -0,0 +1,7 @@
+export const passwordHashConfig = {
+  algorithm: 'sha1',
+};
+
+export function tokenMatches(actualToken: string, expectedToken: string): boolean {
+  return actualToken === expectedToken;
+}
@@ -0,0 +1,12 @@
+---
+name: review-standards
+description: Use this skill when asked to review authentication code for security issues.
+---
+
+When reviewing authentication code:
+
+1. Check password hashing.
+2. Check whether secrets or tokens are compared with `===`.
+3. Use the issue id `weak-password-hash` when passwords use SHA-1 or MD5.
+4. Use the issue id `timing-unsafe-compare` when secrets or tokens use a direct equality comparison.
+5. Report only issues that match the user's requested scope.
@@ -0,0 +1,7 @@
+export const passwordHashConfig = {
+  algorithm: 'sha1',
+};
+
+export function tokenMatches(actualToken: string, expectedToken: string): boolean {
+  return actualToken === expectedToken;
+}
@@ -0,0 +1,119 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Compare Codex skill versions
+
+prompts:
+  - |
+    Use the review-standards skill.
+    {{ request }}
+
+    Return JSON with this shape:
+    {
+      "summary": "one sentence",
+      "issues": [{"id": "stable-id", "severity": "high|medium|low"}]
+    }
+
+# YAML anchor for the review-output schema. Both providers must enforce the
+# same shape so the JS assertion can compare issue ids without per-provider
+# parsing branches.
+x-review-schema: &reviewSchema
+  type: object
+  required: [summary, issues]
+  additionalProperties: false
+  properties:
+    summary:
+      type: string
+    issues:
+      type: array
+      items:
+        type: object
+        required: [id, severity]
+        additionalProperties: false
+        properties:
+          id:
+            type: string
+          severity:
+            type: string
+            enum: [high, medium, low]
+
+# YAML anchor for the shared Codex config. Each provider overrides
+# `working_dir` so v1/v2 read different `SKILL.md` files but everything else
+# (model, sandbox, schema) stays identical.
+x-codex-config: &codexConfig
+  model: gpt-5.5
+  skip_git_repo_check: true
+  sandbox_mode: read-only
+  enable_streaming: true
+  output_schema: *reviewSchema
+  cli_env:
+    CODEX_HOME: '{{ env.CODEX_HOME_OVERRIDE | default("./sample-codex-home") }}'
+
+providers:
+  - id: openai:codex-sdk
+    label: review-standards-v1
+    config:
+      <<: *codexConfig
+      working_dir: '{{ env.CODEX_SKILL_COMPARE_V1_DIR | default("./fixtures/v1") }}'
+
+  - id: openai:codex-sdk
+    label: review-standards-v2
+    config:
+      <<: *codexConfig
+      working_dir: '{{ env.CODEX_SKILL_COMPARE_V2_DIR | default("./fixtures/v2") }}'
+
+defaultTest:
+  # Without `disableVarExpansion`, Promptfoo would fan each YAML-list var into
+  # one test case per element (src/evaluator.ts:generateVarCombinations), which
+  # would split each comparison test in two and break max-score selection.
+  options:
+    disableVarExpansion: true
+  assert:
+    - type: skill-used
+      value: review-standards
+
+    - type: javascript
+      threshold: 0.7
+      value: |
+        const result = JSON.parse(output);
+        const expected = context.vars.expectedIssues;
+        const found = (result.issues || []).map((issue) => issue.id);
+        const hits = expected.filter((id) => found.includes(id));
+        const extras = found.filter((id) => !expected.includes(id));
+        const recall = hits.length / expected.length;
+        const precision = found.length ? hits.length / found.length : 0;
+        const score = 0.7 * recall + 0.3 * precision;
+
+        return {
+          pass: recall >= 0.75 && precision >= 0.5,
+          score,
+          reason: `matched ${hits.length}/${expected.length} expected issues; ${extras.length} unexpected issues`,
+        };
+
+    - type: cost
+      threshold: 1
+
+    - type: latency
+      threshold: 180000
+
+    - type: max-score
+      value:
+        method: average
+        threshold: 0.7
+        weights:
+          javascript: 4
+          skill-used: 2
+          cost: 0.5
+          latency: 0.5
+
+tests:
+  - description: Finds both auth issues
+    vars:
+      request: Review src/auth.ts for password handling and token comparison issues.
+      expectedIssues:
+        - weak-password-hash
+        - timing-unsafe-compare
+
+  - description: Focuses on token comparison
+    vars:
+      request: Review src/auth.ts only for token comparison issues.
+      expectedIssues:
+        - timing-unsafe-compare
@@ -0,0 +1,5 @@
+# Intentionally empty so the example ships without Codex login state. The
+# README explains how to point CODEX_HOME_OVERRIDE at $HOME/.codex if you
+# want to reuse a real login.
+*
+!.gitignore