chore: import upstream snapshot with attribution

test/fixtures/file-script-assertions/comprehensive-llm-rubric-test.yaml

@@ -0,0 +1,300 @@
+# Comprehensive llm-rubric and normal usage test for #6200
+# Tests both file:// scripts AND standard usage patterns
+
+providers:
+  - id: echo
+
+prompts:
+  - '{{prompt}}'
+
+defaultTest:
+  options:
+    provider: echo
+
+tests:
+  # ===========================================
+  # STANDARD USAGE (no file:// - must not break)
+  # ===========================================
+
+  # Test 1: Standard llm-rubric with direct string value
+  - description: 'llm-rubric with direct string value (standard usage)'
+    vars:
+      prompt: 'The capital of France is Paris'
+    assert:
+      - type: llm-rubric
+        value: 'Check that the response correctly identifies Paris as the capital of France'
+
+  # Test 2: Standard llm-rubric with simple criteria
+  - description: 'llm-rubric with simple criteria'
+    vars:
+      prompt: 'Python is a programming language'
+    assert:
+      - type: llm-rubric
+        value: 'The response should mention Python'
+
+  # Test 3: Standard contains with direct value
+  - description: 'contains with direct string (standard usage)'
+    vars:
+      prompt: 'Hello world'
+    assert:
+      - type: contains
+        value: 'Hello'
+
+  # Test 4: Standard equals with direct value
+  - description: 'equals with direct string (standard usage)'
+    vars:
+      prompt: 'exact match'
+    assert:
+      - type: equals
+        value: 'exact match'
+
+  # Test 5: Standard regex with direct pattern
+  - description: 'regex with direct pattern (standard usage)'
+    vars:
+      prompt: 'Order #12345'
+    assert:
+      - type: regex
+        value: "Order #\\d+"
+
+  # Test 6: Standard similar with direct value
+  - description: 'similar with direct value (standard usage)'
+    vars:
+      prompt: 'The weather is nice today'
+    assert:
+      - type: similar
+        value: 'The weather is nice today'
+        threshold: 0.9
+
+  # Test 7: Standard starts-with
+  - description: 'starts-with with direct value'
+    vars:
+      prompt: 'Hello, how are you?'
+    assert:
+      - type: starts-with
+        value: 'Hello'
+
+  # Test 8: Standard contains-all with array
+  - description: 'contains-all with direct array (standard usage)'
+    vars:
+      prompt: 'I love apples and bananas'
+    assert:
+      - type: contains-all
+        value:
+          - apples
+          - bananas
+
+  # Test 9: Standard contains-any
+  - description: 'contains-any with direct array'
+    vars:
+      prompt: 'I prefer oranges'
+    assert:
+      - type: contains-any
+        value:
+          - apples
+          - oranges
+          - bananas
+
+  # ===========================================
+  # LLM-RUBRIC WITH FILE:// SCRIPTS (the fix)
+  # ===========================================
+
+  # Test 10: llm-rubric with JS file:// - simple
+  - description: 'llm-rubric with JS script returning simple string'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 11: llm-rubric with JS file:// - dynamic based on context
+  - description: 'llm-rubric with JS script using context vars'
+    vars:
+      prompt: 'Machine learning is a subset of AI'
+      topic: 'machine learning'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.cjs:rubric
+
+  # Test 12: llm-rubric with JS file:// - object return
+  - description: 'llm-rubric with JS script returning object'
+    vars:
+      prompt: 'Test response'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.cjs:rubricObject
+
+  # Test 13: llm-rubric with Python file://
+  - description: 'llm-rubric with Python script'
+    vars:
+      prompt: 'Neural networks explanation'
+      topic: 'neural networks'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.py:rubric
+
+  # Test 14: llm-rubric with Ruby file://
+  - description: 'llm-rubric with Ruby script'
+    vars:
+      prompt: 'Deep learning concepts'
+      topic: 'deep learning'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.rb:rubric
+
+  # ===========================================
+  # MIXED SCENARIOS
+  # ===========================================
+
+  # Test 15: Multiple assertions - mix of file:// and direct
+  - description: 'Mixed assertions - file:// and direct values'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345 is the answer'
+    assert:
+      - type: contains
+        value: file://rubric-generator.cjs:knownValue
+      - type: contains
+        value: 'answer'
+      - type: starts-with
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 16: llm-rubric alongside other assertions
+  - description: 'llm-rubric with other assertion types'
+    vars:
+      prompt: 'The result is SCRIPT_OUTPUT_12345'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.cjs:knownValue
+      - type: contains
+        value: 'result'
+      - type: regex
+        value: "SCRIPT_OUTPUT_\\d+"
+
+  # ===========================================
+  # OTHER ASSERTIONS WITH FILE://
+  # ===========================================
+
+  # Test 17: equals with file://
+  - description: 'equals with JS file:// script'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+    assert:
+      - type: equals
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 18: contains with file://
+  - description: 'contains with JS file:// script'
+    vars:
+      prompt: 'Result: SCRIPT_OUTPUT_12345'
+    assert:
+      - type: contains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 19: regex with file:// (dynamic pattern)
+  - description: 'regex with file:// script generating pattern'
+    vars:
+      prompt: 'Code: 99999'
+      pattern: "\\d{5}"
+    assert:
+      - type: regex
+        value: file://rubric-generator.cjs:getPattern
+
+  # Test 20: starts-with with file://
+  - description: 'starts-with with file:// script'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345 begins here'
+    assert:
+      - type: starts-with
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 21: icontains with file://
+  - description: 'icontains (case insensitive) with file:// script'
+    vars:
+      prompt: 'script_output_12345 lowercase'
+    assert:
+      - type: icontains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 22: not-contains with file://
+  - description: 'not-contains with file:// script'
+    vars:
+      prompt: 'This has something else'
+    assert:
+      - type: not-contains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 23: contains-all with file:// returning array
+  - description: 'contains-all with file:// array return'
+    vars:
+      prompt: 'Has reference one and reference two'
+    assert:
+      - type: contains-all
+        value: file://rubric-generator.cjs:referenceArray
+
+  # Test 24: contains with file:// numeric return
+  - description: 'contains with file:// numeric return'
+    vars:
+      prompt: 'The answer is 0'
+    assert:
+      - type: contains
+        value: file://rubric-generator.cjs:numericValue
+
+  # ===========================================
+  # EDGE CASES
+  # ===========================================
+
+  # Test 25: Empty string from script
+  - description: 'equals empty string from file:// script'
+    vars:
+      prompt: ''
+    assert:
+      - type: equals
+        value: file://rubric-generator.cjs:emptyValue
+
+  # Test 26: Nunjucks template in direct value
+  - description: 'llm-rubric with nunjucks template'
+    vars:
+      prompt: 'Answer about Paris'
+      city: 'Paris'
+    assert:
+      - type: llm-rubric
+        value: 'Check that the response mentions {{city}}'
+
+  # Test 27: Multiple llm-rubric assertions
+  - description: 'Multiple llm-rubric assertions'
+    vars:
+      prompt: 'Comprehensive answer about AI'
+    assert:
+      - type: llm-rubric
+        value: 'The response should be about AI'
+      - type: llm-rubric
+        value: 'The response should be clear and concise'
+
+  # ===========================================
+  # JAVASCRIPT ASSERTION REGRESSION
+  # ===========================================
+
+  # Test 28: javascript assertion with file:// (uses return as result)
+  - description: 'javascript assertion file:// (regression test)'
+    vars:
+      prompt: 'contains expected keyword'
+    assert:
+      - type: javascript
+        value: file://rubric-generator.cjs:gradingFunction
+
+  # Test 29: javascript assertion inline (regression test)
+  - description: 'javascript assertion inline (regression test)'
+    vars:
+      prompt: 'hello world'
+    assert:
+      - type: javascript
+        value: output.includes("hello")
+
+  # Test 30: javascript assertion with complex logic
+  - description: 'javascript assertion with complex inline logic'
+    vars:
+      prompt: 'The answer is 42'
+    assert:
+      - type: javascript
+        value: |
+          const num = output.match(/\d+/);
+          return num && parseInt(num[0]) === 42;

test/fixtures/file-script-assertions/e2e-test-config.yaml

@@ -0,0 +1,154 @@
+# End-to-end test for script value resolution fix (#6200)
+# Tests that file:// script output is correctly used by assertion handlers
+
+providers:
+  - id: echo
+
+prompts:
+  - '{{prompt}}'
+
+tests:
+  # Test 1: llm-rubric with JavaScript file:// script
+  - description: 'llm-rubric should use JS script output as rubric'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+      topic: 'machine learning'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.cjs:knownValue
+        provider: echo
+
+  # Test 2: contains with JavaScript file:// script
+  - description: 'contains should use JS script output'
+    vars:
+      prompt: 'The answer is SCRIPT_OUTPUT_12345 here'
+    assert:
+      - type: contains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 3: equals with JavaScript file:// script
+  - description: 'equals should use JS script output'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+    assert:
+      - type: equals
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 4: regex with JavaScript file:// script
+  - description: 'regex should use JS script output as pattern'
+    vars:
+      prompt: 'Code: 12345'
+      pattern: "\\d+"
+    assert:
+      - type: regex
+        value: file://rubric-generator.cjs:getPattern
+
+  # Test 5: contains-all with JavaScript file:// script (array return)
+  - description: 'contains-all should use JS script array output'
+    vars:
+      prompt: 'This has reference one and also reference two in it'
+    assert:
+      - type: contains-all
+        value: file://rubric-generator.cjs:referenceArray
+
+  # Test 6: contains with numeric return
+  - description: 'contains should handle numeric script output'
+    vars:
+      prompt: 'The answer is 0'
+    assert:
+      - type: contains
+        value: file://rubric-generator.cjs:numericValue
+
+  # Test 7: javascript assertion with file:// (regression - should still work)
+  - description: 'javascript assertion should use script as grading function'
+    vars:
+      prompt: 'this contains expected word'
+    assert:
+      - type: javascript
+        value: file://rubric-generator.cjs:gradingFunction
+
+  # Test 8: javascript assertion inline (regression - should still work)
+  - description: 'javascript assertion inline should still work'
+    vars:
+      prompt: 'hello world'
+    assert:
+      - type: javascript
+        value: output.includes("hello")
+
+  # Test 9: Direct value (no script) - baseline
+  - description: 'Direct value should work as before'
+    vars:
+      prompt: 'expected text'
+    assert:
+      - type: contains
+        value: 'expected'
+
+  # Test 10: equals with empty string from script
+  - description: 'equals should handle empty string from script'
+    vars:
+      prompt: ''
+    assert:
+      - type: equals
+        value: file://rubric-generator.cjs:emptyValue
+
+  # Test 11: Python script for contains
+  - description: 'contains should use Python script output'
+    vars:
+      prompt: 'The answer is SCRIPT_OUTPUT_12345 here'
+    assert:
+      - type: contains
+        value: file://rubric-generator.py:known_value
+
+  # Test 12: Python script for equals
+  - description: 'equals should use Python script output'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+    assert:
+      - type: equals
+        value: file://rubric-generator.py:known_value
+
+  # Test 13: starts-with with JavaScript script
+  - description: 'starts-with should use JS script output'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345 is the answer'
+    assert:
+      - type: starts-with
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 14: icontains (case insensitive) with script
+  - description: 'icontains should use script output case-insensitively'
+    vars:
+      prompt: 'the answer is script_output_12345 here'
+    assert:
+      - type: icontains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 15: not-contains with script (inverse)
+  - description: 'not-contains should use script output'
+    vars:
+      prompt: 'this does not have the magic value'
+    assert:
+      - type: not-contains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 16: Dynamic rubric with context vars
+  - description: 'llm-rubric should pass context to script'
+    vars:
+      prompt: 'Machine learning explanation here'
+      topic: 'neural networks'
+    assert:
+      - type: llm-rubric
+        value: file://rubric-generator.cjs:rubric
+        provider: echo
+
+  # Test 17: Multiple assertions on same test
+  - description: 'Multiple script-based assertions should all work'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+    assert:
+      - type: equals
+        value: file://rubric-generator.cjs:knownValue
+      - type: contains
+        value: file://rubric-generator.cjs:knownValue
+      - type: starts-with
+        value: file://rubric-generator.cjs:knownValue

test/fixtures/file-script-assertions/failure-test-config.yaml

@@ -0,0 +1,31 @@
+# Test failure scenarios - verify error messages show script output, not file paths
+providers:
+  - id: echo
+
+prompts:
+  - '{{prompt}}'
+
+tests:
+  # Test 1: equals failure - should show script output in error
+  - description: 'equals failure shows script output in error message'
+    vars:
+      prompt: 'wrong value'
+    assert:
+      - type: equals
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 2: contains failure - should show script output in error
+  - description: 'contains failure shows script output in error message'
+    vars:
+      prompt: 'no match here'
+    assert:
+      - type: contains
+        value: file://rubric-generator.cjs:knownValue
+
+  # Test 3: starts-with failure
+  - description: 'starts-with failure shows script output'
+    vars:
+      prompt: 'different start'
+    assert:
+      - type: starts-with
+        value: file://rubric-generator.cjs:knownValue

test/fixtures/file-script-assertions/rubric-generator.cjs

@@ -0,0 +1,68 @@
+// Test fixture for file-based script assertions
+// Used to test that script output is correctly passed to assertion handlers
+
+module.exports.rubric = (_output, context) => {
+  return `Check that the output correctly addresses: ${context.vars.topic}`;
+};
+
+module.exports.expectedValue = () => 'expected string';
+
+// For deterministic testing - script returns a known value
+module.exports.knownValue = () => 'SCRIPT_OUTPUT_12345';
+
+// Returns an object for llm-rubric (which accepts objects)
+module.exports.rubricObject = () => ({
+  role: 'system',
+  content: 'Evaluate the response for accuracy',
+});
+
+// Returns a number for contains assertion
+module.exports.numericValue = () => 0;
+
+// Returns an invalid numeric result for contains assertion
+module.exports.nanValue = () => Number.NaN;
+
+// Returns an array for bleu/gleu assertions
+module.exports.referenceArray = () => ['reference one', 'reference two'];
+
+// Returns empty string
+module.exports.emptyValue = () => '';
+
+// Returns null (edge case)
+module.exports.nullValue = () => null;
+
+// Returns undefined (edge case)
+module.exports.undefinedValue = () => undefined;
+
+// Returns a function (should cause error for non-script assertions)
+module.exports.functionValue = () => () => 'nested function';
+
+// Returns a boolean (should cause error for non-script assertions)
+module.exports.booleanValue = () => true;
+
+// Returns a GradingResult (should cause error for non-script assertions)
+module.exports.gradingResultValue = () => ({
+  pass: true,
+  score: 1,
+  reason: 'This is a grading result',
+});
+
+// Dynamic regex pattern based on context
+module.exports.getPattern = (_output, context) => {
+  return context.vars.pattern || '\\d+';
+};
+
+// For regression testing - javascript assertion that returns GradingResult
+module.exports.gradingFunction = (output, _context) => {
+  const pass = output.includes('expected');
+  return {
+    pass,
+    score: pass ? 1 : 0,
+    reason: pass ? 'Output contains expected' : 'Output missing expected',
+  };
+};
+
+// Returns trace-derived data for regression testing context.trace in non-trace assertion scripts
+module.exports.traceSpanName = (_output, context) => {
+  return context.trace?.spans?.[0]?.name || 'missing-trace';
+};

test/fixtures/file-script-assertions/rubric-generator.py

@@ -0,0 +1,50 @@
+# Test fixture for file-based script assertions
+# Used to test that script output is correctly passed to assertion handlers
+
+
+def rubric(_output, context):
+    return f"Verify the response covers: {context['vars']['topic']}"
+
+
+def expected_value(_output, _context):
+    return "expected string"
+
+
+# For deterministic testing - script returns a known value
+def known_value(_output, _context):
+    return "SCRIPT_OUTPUT_12345"
+
+
+# Returns empty string
+def empty_value(_output, _context):
+    return ""
+
+
+# Returns None (null equivalent)
+def none_value(_output, _context):
+    return None
+
+
+# Dynamic pattern based on context
+def get_pattern(_output, context):
+    return context["vars"].get("pattern", r"\d+")
+
+
+# For regression testing - python assertion that returns boolean
+def grading_function(output, _context):
+    return "expected" in output
+
+
+# Returns a dict/object
+def rubric_object(_output, _context):
+    return {"role": "system", "content": "Evaluate the response for accuracy"}
+
+
+# Returns a number
+def numeric_value(_output, _context):
+    return 42
+
+
+# Returns a list for bleu/gleu
+def reference_array(_output, _context):
+    return ["reference one", "reference two"]

test/fixtures/file-script-assertions/rubric-generator.rb

@@ -0,0 +1,53 @@
+# Test fixture for file-based script assertions
+# Used to test that script output is correctly passed to assertion handlers
+
+def rubric(output, context)
+  "Ensure the answer includes: #{context['vars']['topic']}"
+end
+
+def expected_value(output, context)
+  "expected string"
+end
+
+# For deterministic testing - script returns a known value
+def known_value(output, context)
+  "SCRIPT_OUTPUT_12345"
+end
+
+# Returns empty string
+def empty_value(output, context)
+  ""
+end
+
+# Returns nil (null equivalent)
+def nil_value(output, context)
+  nil
+end
+
+# Dynamic pattern based on context
+def get_pattern(output, context)
+  context['vars']['pattern'] || '\d+'
+end
+
+# For regression testing - ruby assertion that returns boolean
+def grading_function(output, context)
+  output.include?("expected")
+end
+
+# Returns a hash/object
+def rubric_object(output, context)
+  {
+    "role" => "system",
+    "content" => "Evaluate the response for accuracy"
+  }
+end
+
+# Returns a number
+def numeric_value(output, context)
+  42
+end
+
+# Returns an array for bleu/gleu
+def reference_array(output, context)
+  ["reference one", "reference two"]
+end

test/fixtures/file-script-assertions/ruby-test-config.yaml

@@ -0,0 +1,21 @@
+# Ruby script test for script value resolution fix (#6200)
+providers:
+  - id: echo
+
+prompts:
+  - '{{prompt}}'
+
+tests:
+  - description: 'contains should use Ruby script output'
+    vars:
+      prompt: 'The answer is SCRIPT_OUTPUT_12345 here'
+    assert:
+      - type: contains
+        value: file://rubric-generator.rb:known_value
+
+  - description: 'equals should use Ruby script output'
+    vars:
+      prompt: 'SCRIPT_OUTPUT_12345'
+    assert:
+      - type: equals
+        value: file://rubric-generator.rb:known_value