chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,50 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Claude Agent SDK: count LLM turns via gen_ai.turn marker spans'
+
+prompts:
+  - '{{task}}'
+
+providers:
+  # Explicitly allow the tools the tasks below need (pwd via Bash, file listing
+  # via Glob/LS) so they run deterministically in the SDK's isolated temp dir,
+  # rather than relying on default tool availability.
+  - id: anthropic:claude-agent-sdk
+    config:
+      custom_allowed_tools: ['Bash', 'Glob', 'LS']
+
+tracing:
+  enabled: true
+  otlp:
+    http:
+      enabled: true
+      port: 4318
+      host: '127.0.0.1'
+      acceptFormats: ['json']
+
+# Turn counts are model-dependent, so the bounds below are deliberately loose.
+# A single-tool task is typically ~2 rounds (emit the tool, summarize the
+# result) but a model may add a clarification round; a dependent multi-step
+# task takes more. Tighten these only against a pinned model + cached run.
+tests:
+  # Single tool call: a couple of LLM turns (emit the tool, summarize the result).
+  - vars:
+      task: 'Run `pwd` and report the result. Use only one tool call.'
+    metadata:
+      tracingEnabled: true
+    assert:
+      - type: trace-span-count
+        value:
+          pattern: 'gen_ai.turn *'
+          min: 1
+          max: 3
+
+  # Multi-step plan: more turns because each tool call depends on the previous.
+  - vars:
+      task: 'First run `pwd`. Then based on the result, list files in that directory.'
+    metadata:
+      tracingEnabled: true
+    assert:
+      - type: trace-span-count
+        value:
+          pattern: 'gen_ai.turn *'
+          min: 2
@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: 'Claude Agent SDK basic usage - temp directory with no tools'
+
+prompts:
+  - 'Output a Python function that prints the first 10 numbers in the Fibonacci sequence. Do not create any files. Just output the code.'
+
+providers:
+  - anthropic:claude-agent-sdk
+
+tests:
+  - vars: {}
+    assert:
+      - type: contains
+        value: 'fibonacci'
+      - type: llm-rubric
+        value: 'Should generate working Python code that prints first 10 Fibonacci numbers'