chore: import upstream snapshot with attribution

examples/provider-ruby/.gitignore

@@ -0,0 +1,2 @@
+output
+promptfoo-*

examples/provider-ruby/README.md

@@ -0,0 +1,95 @@
+# provider-ruby (Ruby Provider)
+
+This example demonstrates how to create a custom Ruby provider for promptfoo that integrates with the OpenAI API.
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example provider-ruby
+cd provider-ruby
+```
+
+## Overview
+
+The Ruby provider allows you to use Ruby code as a provider in promptfoo evaluations. This example also demonstrates Ruby assertions for custom validation logic.
+
+**Ruby Provider** is useful when you need to:
+
+1. Call APIs from Ruby libraries
+2. Implement custom logic before or after calling LLMs
+3. Process responses in specific ways
+4. Track token usage and other metrics
+
+**Ruby Assertions** allow you to:
+
+1. Write custom validation logic in Ruby
+2. Access test context and variables
+3. Return detailed grading results with scores and reasons
+4. Reuse assertion logic across multiple tests
+
+## Environment Variables
+
+This example requires the following environment variable:
+
+- `OPENAI_API_KEY` - Your OpenAI API key
+
+You can set this in a `.env` file or directly in your environment.
+
+## Requirements
+
+- Ruby 2.7 or higher (with `net/http` and `json` from standard library)
+
+## Files
+
+- `provider.rb` - The Ruby provider implementation that calls OpenAI's API
+- `assert.rb` - Custom Ruby assertion functions for validation
+- `promptfooconfig.yaml` - Configuration for promptfoo evaluation with proper YAML schema reference
+
+## Implementation Details
+
+### Ruby Provider (`provider.rb`)
+
+The Ruby provider includes:
+
+1. A `call_api` function that makes API calls to OpenAI
+2. Token usage extraction from the API response
+3. Multiple sample functions showing different ways to call the API
+
+By default, the example is configured to use `gpt-4.1-mini` model, but you can modify it to use other models as needed.
+
+### Ruby Assertions
+
+The example demonstrates three types of Ruby assertions:
+
+1. **Inline assertions** - Simple one-line checks (e.g., `output.length > 10`)
+2. **Multiline assertions** - Complex logic with detailed results and scores
+3. **External file assertions** (`assert.rb`) - Reusable assertion functions
+
+Ruby assertions can:
+
+- Return boolean values for pass/fail
+- Return numeric scores
+- Return detailed `GradingResult` hashes with pass/fail, score, reason, and component results
+- Access test context including variables, prompts, and provider responses
+
+## Expected Output
+
+When you run this example, you'll see:
+
+1. The prompts being submitted to your Ruby provider
+2. Responses from the OpenAI API
+3. Token usage statistics for each completion
+4. Evaluation results in a table format
+
+Run the example with:
+
+```bash
+npx promptfoo@latest evaluate -c examples/provider-ruby/promptfooconfig.yaml
+```
+
+## Learn More
+
+For more information, see the promptfoo documentation:
+
+- [Ruby Provider](https://promptfoo.dev/docs/providers/ruby/)
+- [Ruby Assertions](https://promptfoo.dev/docs/configuration/expected-outputs/ruby/)

examples/provider-ruby/assert.rb

@@ -0,0 +1,40 @@
+require 'json'
+
+# Default assertion function
+def get_assert(output, context)
+  topic = context['vars']['topic']
+
+  # Check if the output mentions the topic
+  if output.downcase.include?(topic.downcase)
+    {
+      'pass' => true,
+      'score' => 1.0,
+      'reason' => "Output mentions the topic '#{topic}'"
+    }
+  else
+    {
+      'pass' => false,
+      'score' => 0.0,
+      'reason' => "Output does not mention the topic '#{topic}'"
+    }
+  end
+end
+
+# Custom assertion function
+def check_length(output, context)
+  min_length = context.fetch('config', {}).fetch('minLength', 20)
+
+  if output.length >= min_length
+    {
+      'pass' => true,
+      'score' => 1.0,
+      'reason' => "Output length #{output.length} meets minimum #{min_length}"
+    }
+  else
+    {
+      'pass' => false,
+      'score' => output.length.to_f / min_length,
+      'reason' => "Output length #{output.length} is below minimum #{min_length}"
+    }
+  end
+end

examples/provider-ruby/promptfooconfig.yaml

@@ -0,0 +1,47 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Custom Ruby provider with functions
+
+prompts:
+  - 'Write a very concise funny tweet about {{topic}}'
+
+providers:
+  - id: file://provider.rb
+    config:
+      someOption: foobar
+
+  - id: file://provider.rb:some_other_function
+
+tests:
+  - vars:
+      topic: bananas
+    assert:
+      - type: contains
+        value: Bananamax
+      - type: ruby
+        value: output.length > 10
+  - vars:
+      topic: fruits
+    assert:
+      - type: llm-rubric
+        value: includes at least one emoji
+      - type: ruby
+        value: file://assert.rb
+      - type: ruby
+        value: file://assert.rb:check_length
+        config:
+          minLength: 15
+  - vars:
+      topic: turtles
+    assert:
+      - type: llm-rubric
+        value: is funny
+      - type: ruby
+        value: |
+          # Check if output contains specific words
+          words = ['turtle', 'shell', 'slow']
+          count = words.count { |word| output.downcase.include?(word) }
+          {
+            'pass' => count >= 1,
+            'score' => count / 3.0,
+            'reason' => "Found #{count} of #{words.length} turtle-related words"
+          }

examples/provider-ruby/provider.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'json'
+require 'uri'
+
+##
+# Sends the prompt to OpenAI's chat completion endpoint with a system role of a marketer for "Bananamax" and returns the assistant's output along with token usage and metadata.
+# @param [String] prompt - The user-facing prompt to submit to the model.
+# @param [Hash] options - Optional settings; may include a 'config' key (Hash) which will be returned under metadata.
+# @param [Hash] context - Additional contextual information provided by the caller (preserved but not included in the request body).
+# @return [Hash] A hash containing:
+#   - 'output' => String: the assistant message content from the first choice.
+#   - 'tokenUsage' => Hash or nil: when present, contains 'total', 'prompt', and 'completion' token counts.
+#   - 'metadata' => Hash: includes 'config' reflecting options['config'] or an empty hash.
+def call_api(prompt, options, context)
+  # Get config values
+  config = options['config'] || {}
+
+  # Prepare API request
+  uri = URI.parse('https://api.openai.com/v1/chat/completions')
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = true
+
+  request = Net::HTTP::Post.new(uri.path)
+  request['Content-Type'] = 'application/json'
+  request['Authorization'] = "Bearer #{ENV['OPENAI_API_KEY']}"
+
+  request.body = JSON.generate({
+    'model' => 'gpt-4.1-mini',
+    'messages' => [
+      {
+        'role' => 'system',
+        'content' => 'You are a marketer working for a startup called Bananamax.'
+      },
+      {
+        'role' => 'user',
+        'content' => prompt
+      }
+    ]
+  })
+
+  # Make API call
+  response = http.request(request)
+  data = JSON.parse(response.body)
+
+  # Extract token usage information from the response
+  token_usage = nil
+  if data['usage']
+    token_usage = {
+      'total' => data['usage']['total_tokens'],
+      'prompt' => data['usage']['prompt_tokens'],
+      'completion' => data['usage']['completion_tokens']
+    }
+  end
+
+  {
+    'output' => data['choices'][0]['message']['content'],
+    'tokenUsage' => token_usage,
+    'metadata' => {
+      'config' => config
+    }
+  }
+end
+
+##
+# Appends an instruction to the prompt to produce an all-caps response and forwards the request to the API.
+# @param [String] prompt - The user prompt to send.
+# @param [Hash] options - Request options; may include a `config` hash used in returned metadata.
+# @param [Hash] context - Additional contextual data passed through to the API call.
+# @return [Hash] A hash with keys:
+#   - 'output' => the assistant's message content,
+#   - 'tokenUsage' => a hash with `total`, `prompt`, and `completion` token counts or `nil`,
+#   - 'metadata' => a hash containing the provided `config`.
+def some_other_function(prompt, options, context)
+  call_api(prompt + "\nWrite in ALL CAPS", options, context)
+end
+
+# Example usage when running the script directly
+if __FILE__ == $PROGRAM_NAME
+  prompt = 'What is the weather in San Francisco?'
+  options = { 'config' => { 'optionFromYaml' => 123 } }
+  context = { 'vars' => { 'location' => 'San Francisco' } }
+
+  puts JSON.pretty_generate(call_api(prompt, options, context))
+end