comet-ml--opik/apps/opik-python-backend/tests/unit/test_metrics_factory.py

"""Tests for MetricFactory in Optimization Studio."""

import pytest
from unittest.mock import MagicMock, patch

from opik.evaluation.metrics.score_result import ScoreResult
from opik_backend.studio.metrics import MetricFactory
from opik_backend.studio.exceptions import InvalidMetricError
from opik_backend.studio.types import _convert_template_syntax, OptimizationConfig


class TestMetricFactory:
    """Tests for MetricFactory.build() and metric builders."""

    def test_build_unknown_metric_raises_error(self):
        """Test that building an unknown metric type raises InvalidMetricError."""
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build("unknown_metric", {}, "openai/gpt-4o")
        
        assert "unknown_metric" in str(exc_info.value)
        assert "Available metrics:" in str(exc_info.value)

    def test_build_equals_metric(self):
        """Test building an equals metric."""
        metric_fn = MetricFactory.build("equals", {}, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "equals"
        assert callable(metric_fn)

    def test_build_equals_metric_with_params(self):
        """Test building an equals metric with custom parameters."""
        params = {
            "case_sensitive": False,
            "reference_key": "expected_output"
        }
        metric_fn = MetricFactory.build("equals", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "equals"
        assert callable(metric_fn)

    def test_build_levenshtein_metric(self):
        """Test building a levenshtein_ratio metric."""
        metric_fn = MetricFactory.build("levenshtein_ratio", {}, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "levenshtein_ratio"
        assert callable(metric_fn)

    def test_build_geval_metric(self):
        """Test building a geval metric."""
        params = {
            "task_introduction": "Evaluate the response quality",
            "evaluation_criteria": "Is the response helpful?"
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)

    def test_build_json_schema_validator_metric(self):
        """Test building a json_schema_validator metric."""
        # The metric reads schema from dataset items via schema_key parameter
        metric_fn = MetricFactory.build("json_schema_validator", {}, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "json_schema_validator"
        assert callable(metric_fn)

    def test_build_json_schema_validator_metric_with_custom_schema_key(self):
        """Test building a json_schema_validator metric with custom schema_key."""
        params = {"schema_key": "my_schema"}
        metric_fn = MetricFactory.build("json_schema_validator", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "json_schema_validator"
        assert callable(metric_fn)

    def test_json_schema_validator_missing_schema_returns_zero(self):
        """Test that json_schema_validator returns 0.0 when schema is missing from dataset item."""
        metric_fn = MetricFactory.build("json_schema_validator", {}, "openai/gpt-4o")
        
        # Dataset item without json_schema key
        dataset_item = {"other_field": "value"}
        result = metric_fn(dataset_item, '{"name": "test"}')
        
        assert result.value == 0.0
        assert "Missing schema" in result.reason


class TestEqualsMetricExecution:
    """Tests for equals metric function execution."""

    def test_equals_metric_exact_match(self):
        """Test equals metric with exact match."""
        metric_fn = MetricFactory.build("equals", {"case_sensitive": True}, "model")
        
        # Default reference key is "answer"
        dataset_item = {"answer": "hello world"}
        result = metric_fn(dataset_item, "hello world")
        
        assert result.value == 1.0

    def test_equals_metric_no_match(self):
        """Test equals metric with no match."""
        metric_fn = MetricFactory.build("equals", {"case_sensitive": True}, "model")
        
        dataset_item = {"answer": "hello world"}
        result = metric_fn(dataset_item, "goodbye world")
        
        assert result.value == 0.0

    def test_equals_metric_case_insensitive(self):
        """Test equals metric with case insensitive comparison."""
        metric_fn = MetricFactory.build("equals", {"case_sensitive": False}, "model")
        
        dataset_item = {"answer": "Hello World"}
        result = metric_fn(dataset_item, "hello world")
        
        assert result.value == 1.0

    def test_equals_metric_custom_reference_key(self):
        """Test equals metric with custom reference key."""
        metric_fn = MetricFactory.build(
            "equals", 
            {"reference_key": "expected"}, 
            "model"
        )
        
        dataset_item = {"expected": "test value"}
        result = metric_fn(dataset_item, "test value")
        
        assert result.value == 1.0


class TestLevenshteinMetricExecution:
    """Tests for levenshtein_ratio metric function execution."""

    def test_levenshtein_metric_exact_match(self):
        """Test levenshtein metric with exact match."""
        metric_fn = MetricFactory.build("levenshtein_ratio", {}, "model")
        
        # Default reference key is "answer"
        dataset_item = {"answer": "hello"}
        result = metric_fn(dataset_item, "hello")
        
        assert result.value == 1.0

    def test_levenshtein_metric_partial_match(self):
        """Test levenshtein metric with partial match."""
        metric_fn = MetricFactory.build("levenshtein_ratio", {}, "model")
        
        dataset_item = {"answer": "hello"}
        result = metric_fn(dataset_item, "hallo")
        
        # "hello" vs "hallo" - 1 character difference out of 5
        assert 0.0 < result.value < 1.0

    def test_levenshtein_metric_no_match(self):
        """Test levenshtein metric with completely different strings."""
        metric_fn = MetricFactory.build("levenshtein_ratio", {}, "model")
        
        dataset_item = {"answer": "abc"}
        result = metric_fn(dataset_item, "xyz")
        
        assert result.value == 0.0


class TestMetricReasons:
    """Tests for metric reason fields (required for hierarchical_reflective optimizer)."""

    def test_equals_metric_includes_reason_on_match(self):
        """Test equals metric includes reason field on match."""
        metric_fn = MetricFactory.build("equals", {}, "model")
        dataset_item = {"answer": "test"}
        result = metric_fn(dataset_item, "test")
        
        assert result.reason is not None
        assert "match" in result.reason.lower()

    def test_equals_metric_includes_reason_on_no_match(self):
        """Test equals metric includes reason field on no match."""
        metric_fn = MetricFactory.build("equals", {}, "model")
        dataset_item = {"answer": "test"}
        result = metric_fn(dataset_item, "different")
        
        assert result.reason is not None
        assert "no match" in result.reason.lower()

    def test_levenshtein_metric_includes_reason(self):
        """Test levenshtein metric includes reason field with similarity percentage."""
        metric_fn = MetricFactory.build("levenshtein_ratio", {}, "model")
        dataset_item = {"answer": "hello"}
        result = metric_fn(dataset_item, "hallo")
        
        assert result.reason is not None
        assert "similarity" in result.reason.lower()
        assert "%" in result.reason


class TestCodeMetric:
    """Tests for code metric functionality.
    
    Code metrics use the same executor infrastructure as automations (evaluation metrics),
    executed via ProcessExecutor or DockerExecutor based on PYTHON_CODE_EXECUTOR_STRATEGY.
    
    Only BaseMetric class pattern is supported (same as automations).
    """

    def test_code_metric_basic_class_works(self):
        """Test that a basic class metric works."""
        code = '''
from opik.evaluation.metrics import BaseMetric
from opik.evaluation.metrics.score_result import ScoreResult

class MyMetric(BaseMetric):
    def __init__(self, name: str = "test"):
        super().__init__(name=name)
    
    def score(self, output, **kwargs):
        return ScoreResult(name=self.name, value=0.5, reason="Class metric")
'''
        metric_fn = MetricFactory.build("code", {"code": code}, "model")
        
        result = metric_fn({}, "test output")
        assert result.value == 0.5
        assert result.name == "test"

    def test_code_metric_uses_json(self):
        """Test that json module can be used."""
        code = '''
import json
from opik.evaluation.metrics import BaseMetric
from opik.evaluation.metrics.score_result import ScoreResult

class JsonMetric(BaseMetric):
    def __init__(self, name: str = "json_test"):
        super().__init__(name=name)
    
    def score(self, output, **kwargs):
        data = json.loads(output) if output.startswith("{") else {}
        return ScoreResult(name=self.name, value=1.0, reason="Used json")
'''
        metric_fn = MetricFactory.build("code", {"code": code}, "model")
        result = metric_fn({}, '{"key": "value"}')
        assert result.value == 1.0

    def test_code_metric_uses_re(self):
        """Test that re module can be used."""
        code = '''
import re
from opik.evaluation.metrics import BaseMetric
from opik.evaluation.metrics.score_result import ScoreResult

class RegexMetric(BaseMetric):
    def __init__(self, name: str = "regex_test"):
        super().__init__(name=name)
    
    def score(self, output, **kwargs):
        match = re.search(r"\\d+", output)
        return ScoreResult(name=self.name, value=1.0 if match else 0.0, reason="Used re")
'''
        metric_fn = MetricFactory.build("code", {"code": code}, "model")
        result = metric_fn({}, "test 123")
        assert result.value == 1.0

    def test_code_metric_uses_math(self):
        """Test that math module can be used."""
        code = '''
import math
from opik.evaluation.metrics import BaseMetric
from opik.evaluation.metrics.score_result import ScoreResult

class MathMetric(BaseMetric):
    def __init__(self, name: str = "math_test"):
        super().__init__(name=name)
    
    def score(self, output, **kwargs):
        return ScoreResult(name=self.name, value=math.sqrt(0.25), reason="Used math")
'''
        metric_fn = MetricFactory.build("code", {"code": code}, "model")
        result = metric_fn({}, "test")
        assert result.value == 0.5

    def test_code_metric_receives_dataset_fields_as_kwargs(self):
        """Test that dataset_item fields are passed as kwargs to score method."""
        code = '''
from opik.evaluation.metrics import BaseMetric
from opik.evaluation.metrics.score_result import ScoreResult

class KwargsMetric(BaseMetric):
    def __init__(self, name: str = "kwargs_test"):
        super().__init__(name=name)
    
    def score(self, output, **kwargs):
        expected = kwargs.get("expected_value", "")
        score = 1.0 if output == expected else 0.0
        return ScoreResult(name=self.name, value=score, reason=f"Expected: {expected}")
'''
        metric_fn = MetricFactory.build("code", {"code": code}, "model")
        
        # Test with matching expected_value
        result = metric_fn({"expected_value": "correct"}, "correct")
        assert result.value == 1.0
        
        # Test with non-matching expected_value
        result = metric_fn({"expected_value": "correct"}, "wrong")
        assert result.value == 0.0

    def test_code_metric_preserves_custom_name(self):
        """Test that the metric name defined by user is preserved."""
        code = '''
from opik.evaluation.metrics import BaseMetric
from opik.evaluation.metrics.score_result import ScoreResult

class CustomNamedMetric(BaseMetric):
    def __init__(self, name: str = "my_custom_metric_name"):
        super().__init__(name=name)
    
    def score(self, output, **kwargs):
        return ScoreResult(name=self.name, value=1.0, reason="Test")
'''
        metric_fn = MetricFactory.build("code", {"code": code}, "model")
        result = metric_fn({}, "test output")
        
        assert result.name == "my_custom_metric_name"

    def test_code_metric_missing_code_raises_error(self):
        """Test that missing code parameter raises error."""
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build("code", {}, "model")
        
        assert "Missing 'code' parameter" in str(exc_info.value)

    def test_code_metric_empty_code_raises_error(self):
        """Test that empty code raises error."""
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build("code", {"code": ""}, "model")
        
        assert "Missing 'code' parameter" in str(exc_info.value)

    def test_code_metric_invalid_syntax_raises_error(self):
        """Test that invalid Python syntax raises error."""
        code = '''
class MyMetric(BaseMetric)
    def score(self, output, **kwargs):
        return ScoreResult(name="test", value=1.0, reason="OK")
'''
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build("code", {"code": code}, "model")
        
        assert "Invalid Python code" in str(exc_info.value)

    def test_code_metric_no_basemetric_class_raises_error(self):
        """Test that code without a BaseMetric subclass raises error at build time.
        
        With executor infrastructure, code must define a BaseMetric subclass.
        Validation at build time provides fail-fast behavior.
        """
        code = '''
# Just a comment, no BaseMetric class
x = 1
'''
        # Should raise InvalidMetricError during build (validation step)
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build("code", {"code": code}, "model")
        
        assert "BaseMetric" in str(exc_info.value)

    def test_code_metric_function_only_raises_error(self):
        """Test that function-only code (no BaseMetric class) raises error at build time.
        
        Function-based metrics are not supported - only BaseMetric class pattern.
        Validation at build time provides fail-fast behavior.
        """
        code = '''
from opik.evaluation.metrics.score_result import ScoreResult

def my_metric(dataset_item, llm_output):
    return ScoreResult(name="test", value=1.0, reason="Function")
'''
        # Should raise InvalidMetricError during build (validation step)
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build("code", {"code": code}, "model")
        
        assert "BaseMetric" in str(exc_info.value)


class TestJsonPathReferenceKey:
    """Tests for JSONPath support in reference_key for equals and levenshtein metrics."""

    def test_equals_jsonpath_filter_expression(self):
        """Test equals metric with a JSONPath filter to extract a value from a JSON array."""
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "$.feedback_scores[?(@.name == 'Useful')].value"},
            "model",
        )

        dataset_item = {
            "feedback_scores": [
                {"name": "Usefulness", "value": 0.8, "source": "ONLINE_SCORING"},
                {"name": "Useful", "category_name": "Fail", "value": 0, "source": "UI"},
                {"name": "Useful-Numerical", "value": 2, "source": "UI"},
            ]
        }
        result = metric_fn(dataset_item, "0")
        assert result.value == 1.0

    def test_levenshtein_jsonpath_filter_expression(self):
        """Test levenshtein metric with a JSONPath filter expression."""
        metric_fn = MetricFactory.build(
            "levenshtein_ratio",
            {"reference_key": "$.feedback_scores[?(@.name == 'Useful')].category_name"},
            "model",
        )

        dataset_item = {
            "feedback_scores": [
                {"name": "Usefulness", "category_name": "Pass", "value": 0.8},
                {"name": "Useful", "category_name": "Fail", "value": 0},
            ]
        }
        result = metric_fn(dataset_item, "Fail")
        assert result.value == 1.0

    def test_jsonpath_index_access(self):
        """Test reference_key with a JSONPath array index."""
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "$.scores[0].value"},
            "model",
        )

        dataset_item = {
            "scores": [
                {"name": "first", "value": "42"},
                {"name": "second", "value": "99"},
            ]
        }
        result = metric_fn(dataset_item, "42")
        assert result.value == 1.0

    def test_jsonpath_no_match_scores_zero_with_reason(self):
        """A JSONPath that resolves nothing must not silently match empty output.

        Regression guard for OPIK-7160: the old behavior defaulted an
        unresolvable reference to "" and reported a perfect 1.0 against empty
        output, hiding the misconfiguration. It now scores 0.0 and explains why.
        """
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "$.feedback_scores[?(@.name == 'NonExistent')].value"},
            "model",
        )

        dataset_item = {
            "feedback_scores": [
                {"name": "Useful", "value": 0},
            ]
        }
        result = metric_fn(dataset_item, "")
        assert result.value == 0.0
        assert "Missing reference value" in result.reason

    def test_jsonpath_no_match_against_nonempty_output(self):
        """A JSONPath with no matches scores 0 against non-empty output."""
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "$.feedback_scores[?(@.name == 'NonExistent')].value"},
            "model",
        )

        dataset_item = {
            "feedback_scores": [
                {"name": "Useful", "value": 0},
            ]
        }
        result = metric_fn(dataset_item, "something")
        assert result.value == 0.0

    def test_plain_key_still_works(self):
        """Test that plain field names continue to work as before."""
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "answer"},
            "model",
        )

        dataset_item = {"answer": "hello"}
        result = metric_fn(dataset_item, "hello")
        assert result.value == 1.0

    def test_is_jsonpath_detection(self):
        """Test the heuristic that distinguishes plain keys from JSONPath expressions."""
        from opik_backend.studio.metrics import _is_jsonpath

        assert _is_jsonpath("answer") is False
        assert _is_jsonpath("expected_output") is False
        assert _is_jsonpath("my-field") is False
        assert _is_jsonpath("my.field") is False

        assert _is_jsonpath("$.answer") is True
        assert _is_jsonpath("scores[0].value") is True
        assert _is_jsonpath("$.scores[?(@.name == 'x')].value") is True
        assert _is_jsonpath("items..value") is True


class TestReferenceKeyValidation:
    """Build-time validation that a reference_key resolves against the dataset.

    Guards OPIK-7160: a reference_key matching no dataset field silently scored
    every item 0, so no candidate could beat the baseline and the optimizer
    returned the seed prompt while the run reported "completed". Building the
    metric now fails loudly instead, keeping that failure distinguishable from a
    legitimate "no improvement over baseline" run (OPIK-7038).
    """

    def test_equals_build_raises_when_key_resolves_no_item(self):
        dataset_items = [{"answer": "a"}, {"answer": "b"}]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "equals",
                {"reference_key": "label"},  # not a dataset field
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        message = str(exc_info.value)
        assert "label" in message
        assert "did not resolve" in message
        # Available fields are surfaced to make the fix obvious.
        assert "answer" in message

    def test_levenshtein_build_raises_when_key_resolves_no_item(self):
        dataset_items = [{"answer": "a"}, {"answer": "b"}]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "levenshtein_ratio",
                {"reference_key": "typo"},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        assert "did not resolve" in str(exc_info.value)

    def test_numerical_similarity_build_raises_when_key_resolves_no_item(self):
        dataset_items = [{"score": 1}, {"score": 2}]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "numerical_similarity",
                {"reference_key": "value"},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        assert "did not resolve" in str(exc_info.value)

    def test_build_passes_when_key_resolves_for_some_items(self):
        # Sparse data: the key is present on only one item. Validation passes;
        # missing items are handled per-item at scoring time.
        dataset_items = [{"answer": "a"}, {"other": "b"}]
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "answer"},
            "model",
            dataset_items_provider=lambda: dataset_items,
        )
        assert callable(metric_fn)

    def test_build_skips_validation_without_provider(self):
        # No dataset available (e.g. config validation) -> do not guess, skip.
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "anything"},
            "model",
        )
        assert callable(metric_fn)

    def test_build_skips_validation_for_empty_dataset(self):
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "anything"},
            "model",
            dataset_items_provider=lambda: [],
        )
        assert callable(metric_fn)

    def test_equals_jsonpath_build_raises_when_no_item_matches(self):
        dataset_items = [
            {"feedback_scores": [{"name": "Useful", "value": 0}]},
        ]
        with pytest.raises(InvalidMetricError):
            MetricFactory.build(
                "equals",
                {"reference_key": "$.feedback_scores[?(@.name == 'Missing')].value"},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )

    def test_malformed_jsonpath_build_raises_with_syntax_error(self):
        # A JSONPath-shaped key with invalid syntax that matches no literal field
        # must fail with a JSONPath-specific message, not the generic
        # "did not resolve" one (which hides the real cause). The literal
        # fallback in _resolve_reference otherwise swallows the parse error.
        dataset_items = [{"answer": "42"}]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "equals",
                {"reference_key": "$.foo["},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        assert "not a valid JSONPath expression" in str(exc_info.value)

    def test_numerical_similarity_malformed_jsonpath_build_raises(self):
        # Same guard on the numerical_similarity validation path, which infers
        # scale from a separate resolution loop.
        dataset_items = [{"score": 3}]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "numerical_similarity",
                {"reference_key": "$.foo["},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        assert "not a valid JSONPath expression" in str(exc_info.value)

    def test_build_passes_when_field_present_but_null(self):
        # A field that exists on every item but holds null is a data-quality
        # issue, not a key misconfiguration -> the metric must still build
        # (regression guard: this previously hard-failed the run with a
        # self-contradictory "did not resolve ... available fields: answer"
        # message).
        dataset_items = [{"answer": None}, {"answer": None}]
        metric_fn = MetricFactory.build(
            "equals",
            {"reference_key": "answer"},
            "model",
            dataset_items_provider=lambda: dataset_items,
        )
        assert callable(metric_fn)

    def test_build_does_not_crash_on_non_dict_items(self):
        # Malformed dataset items must yield a clean InvalidMetricError, not an
        # AttributeError from .get()/.keys() on a non-dict.
        dataset_items = [None, "scalar", 42]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "equals",
                {"reference_key": "answer"},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        assert "did not resolve" in str(exc_info.value)

    def test_numerical_similarity_raises_when_references_non_numeric(self):
        # The key resolves for every item, but to non-numeric text -> every item
        # would score 0 (silent flat-0). numerical_similarity must fail loudly
        # (OPIK-7160), unlike equals/levenshtein for which any resolved value is
        # scoreable.
        dataset_items = [{"answer": "positive"}, {"answer": "negative"}]
        with pytest.raises(InvalidMetricError) as exc_info:
            MetricFactory.build(
                "numerical_similarity",
                {"reference_key": "answer"},
                "model",
                dataset_items_provider=lambda: dataset_items,
            )
        assert "no dataset item held a numeric value" in str(exc_info.value)

    def test_numerical_similarity_builds_with_one_numeric_reference(self):
        # Sparse numeric data: only some items are numeric. Build succeeds
        # (there is at least one numeric reference to work with) and the numeric
        # scoring path is actually live -- not a silent flat-0.
        dataset_items = [{"score": 3}, {"score": "n/a"}]
        metric_fn = MetricFactory.build(
            "numerical_similarity",
            {"reference_key": "score"},
            "model",
            dataset_items_provider=lambda: dataset_items,
        )
        assert callable(metric_fn)
        # A single numeric reference means scale_range falls back to 1.0, so an
        # exact match scores 1.0 -- exercising the similarity math, not just the
        # constructor.
        exact = metric_fn({"score": 3}, "3")
        assert exact.value == 1.0
        assert exact.name == "numerical_similarity"
        # A unit-off output is normalized against scale_range=1.0 -> 0.0.
        off_by_one = metric_fn({"score": 3}, "2")
        assert off_by_one.value == 0.0
        # The non-numeric sibling still yields a clean 0 with an explanatory reason.
        non_numeric = metric_fn({"score": "n/a"}, "3")
        assert non_numeric.value == 0.0
        assert "not numeric" in non_numeric.reason


class TestMissingReferencePerItem:
    """Per-item feedback when a reference key is absent on a specific item."""

    def test_equals_missing_reference_scores_zero_with_reason(self):
        metric_fn = MetricFactory.build("equals", {"reference_key": "answer"}, "model")
        result = metric_fn({"other": "x"}, "x")
        assert result.value == 0.0
        assert "Missing reference value" in result.reason
        assert "answer" in result.reason

    def test_levenshtein_missing_reference_scores_zero_with_reason(self):
        metric_fn = MetricFactory.build(
            "levenshtein_ratio", {"reference_key": "answer"}, "model"
        )
        result = metric_fn({"other": "x"}, "x")
        assert result.value == 0.0
        assert "Missing reference value" in result.reason

    def test_present_empty_string_reference_matches_empty_output(self):
        # A field that is present but holds "" is a real reference value, not a
        # missing one: empty output should still score a perfect match. Only a
        # genuinely absent field short-circuits to the missing-reference result.
        metric_fn = MetricFactory.build("equals", {"reference_key": "answer"}, "model")
        result = metric_fn({"answer": ""}, "")
        assert result.value == 1.0


class TestNumericalSimilarityMetric:

    def test_exact_match(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        result = metric_fn({"score": 0.7}, "0.7")
        assert result.value == 1.0
        assert result.name == "numerical_similarity"

    def test_close_values(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        result = metric_fn({"score": 0.7}, "0.85")
        # scale_range=1.0 (no dataset), diff=0.15 -> max(0, 1 - 0.15) = 0.85
        assert abs(result.value - 0.85) < 1e-6

    def test_far_values_clamps_to_zero(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        result = metric_fn({"score": 0.0}, "5.0")
        # scale_range=1.0 (no dataset), diff=5.0 -> max(0, 1 - 5.0) = 0.0
        assert result.value == 0.0

    def test_non_numeric_output(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        result = metric_fn({"score": 0.7}, "not a number")
        assert result.value == 0.0
        assert "Could not parse" in result.reason

    def test_missing_reference(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "missing"}, "model")
        result = metric_fn({"score": 0.7}, "0.7")
        assert result.value == 0.0
        assert "Missing reference" in result.reason

    def test_non_numeric_reference(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        result = metric_fn({"score": "not a number"}, "0.7")
        assert result.value == 0.0
        assert "not numeric" in result.reason

    def test_with_jsonpath_reference(self):
        metric_fn = MetricFactory.build(
            "numerical_similarity",
            {"reference_key": "$.feedback_scores[?(@.name == 'Useful')].value"},
            "model",
        )
        dataset_item = {
            "feedback_scores": [
                {"name": "Useful", "value": 0.7},
                {"name": "Usefulness", "value": 0.8},
            ]
        }
        result = metric_fn(dataset_item, "0.85")
        # scale_range=1.0 (no dataset), diff=0.15 -> max(0, 1 - 0.15) = 0.85
        assert abs(result.value - 0.85) < 1e-6

    def test_integer_values(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        result = metric_fn({"score": 1}, "1")
        assert result.value == 1.0

    def test_scale_range_inferred_from_dataset(self):
        dataset_items = [{"score": 0}, {"score": 1}, {"score": 2}, {"score": 3}, {"score": 4}, {"score": 5}]
        metric_fn = MetricFactory.build(
            "numerical_similarity", {"reference_key": "score"}, "model",
            dataset_items_provider=lambda: dataset_items,
        )
        # scale_range = 5 - 0 = 5
        # ref=4.5, output=4 -> normalized_error = 0.5/5 = 0.1 -> max(0, 1 - 0.1) = 0.9
        result = metric_fn({"score": 4.5}, "4")
        assert abs(result.value - 0.9) < 1e-6

    def test_scale_range_max_error_gives_zero(self):
        dataset_items = [{"score": 0}, {"score": 5}]
        metric_fn = MetricFactory.build(
            "numerical_similarity", {"reference_key": "score"}, "model",
            dataset_items_provider=lambda: dataset_items,
        )
        # scale_range=5, diff=5 -> normalized_error=1.0 -> max(0, 1-1) = 0.0
        result = metric_fn({"score": 0}, "5")
        assert result.value == 0.0

    def test_scale_range_fallback_without_dataset(self):
        metric_fn = MetricFactory.build("numerical_similarity", {"reference_key": "score"}, "model")
        # No provider -> scale_range=1.0 -> max(0, 1 - 0.5) = 0.5
        result = metric_fn({"score": 4.5}, "4")
        assert abs(result.value - 0.5) < 1e-6

    def test_scale_range_single_value_no_range(self):
        dataset_items = [{"score": 3}, {"score": 3}, {"score": 3}]
        metric_fn = MetricFactory.build(
            "numerical_similarity", {"reference_key": "score"}, "model",
            dataset_items_provider=lambda: dataset_items,
        )
        # All same value -> range=0 -> falls back to scale_range=1.0
        # diff=1 -> max(0, 1 - 1.0) = 0.0
        result = metric_fn({"score": 3}, "4")
        assert result.value == 0.0


class TestGEvalTemplateInterpolation:
    """Tests for GEval metric template interpolation with dataset item fields."""

    def test_geval_with_none_params_uses_defaults(self):
        """Test that GEval handles explicit None params without crashing."""
        # Callers may explicitly pass None for optional fields
        params = {
            "task_introduction": None,
            "evaluation_criteria": None
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)

    def test_geval_with_missing_params_uses_defaults(self):
        """Test that GEval handles missing params using defaults."""
        params = {}
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)

    def test_geval_with_placeholders_handles_none_dataset_item(self):
        """Test that GEval with placeholders handles None dataset_item without crashing."""
        from opik_backend.studio.metrics import _interpolate_template
        
        params = {
            "task_introduction": "Evaluate the {{topic}} response",
            "evaluation_criteria": "Check if output matches {{answer}}"
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)
        
        # Verify interpolation with None returns template unchanged (placeholders preserved)
        result = _interpolate_template(params["evaluation_criteria"], {})
        assert result == "Check if output matches {{answer}}"

    def test_geval_without_placeholders_creates_single_instance(self):
        """Test that GEval without placeholders creates a single reusable instance."""
        from opik_backend.studio.metrics import _interpolate_template
        
        params = {
            "task_introduction": "Evaluate the response quality",
            "evaluation_criteria": "Is the response helpful and accurate?"
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)
        
        # Verify no interpolation changes static text
        dataset_item = {"answer": "42"}
        assert _interpolate_template(params["task_introduction"], dataset_item) == "Evaluate the response quality"
        assert _interpolate_template(params["evaluation_criteria"], dataset_item) == "Is the response helpful and accurate?"

    def test_geval_with_placeholders_in_criteria(self):
        """Test that GEval with {{field}} placeholders in criteria works."""
        from opik_backend.studio.metrics import _interpolate_template
        
        params = {
            "task_introduction": "Evaluate the response",
            "evaluation_criteria": "Check if the output matches the expected answer: {{answer}}"
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)
        
        # Verify placeholder interpolation in criteria
        dataset_item = {"answer": "42"}
        result = _interpolate_template(params["evaluation_criteria"], dataset_item)
        assert result == "Check if the output matches the expected answer: 42"

    def test_geval_with_placeholders_in_task_introduction(self):
        """Test that GEval with {{field}} placeholders in task_introduction works."""
        from opik_backend.studio.metrics import _interpolate_template
        
        params = {
            "task_introduction": "You are evaluating a {{topic}} question",
            "evaluation_criteria": "Is the response accurate?"
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)
        
        # Verify placeholder interpolation in task_introduction
        dataset_item = {"topic": "math"}
        result = _interpolate_template(params["task_introduction"], dataset_item)
        assert result == "You are evaluating a math question"

    def test_geval_with_multiple_placeholders(self):
        """Test that GEval with multiple {{field}} placeholders works."""
        from opik_backend.studio.metrics import _interpolate_template
        
        params = {
            "task_introduction": "Evaluate the {{task_type}} response for {{domain}}",
            "evaluation_criteria": "Expected answer is {{answer}}. Context: {{context}}"
        }
        metric_fn = MetricFactory.build("geval", params, "openai/gpt-4o")
        
        assert metric_fn.__name__ == "geval"
        assert callable(metric_fn)
        
        # Verify multiple placeholder interpolation
        dataset_item = {
            "task_type": "homework",
            "domain": "algebra", 
            "answer": "x=5",
            "context": "solving equations"
        }
        intro_result = _interpolate_template(params["task_introduction"], dataset_item)
        criteria_result = _interpolate_template(params["evaluation_criteria"], dataset_item)
        
        assert intro_result == "Evaluate the homework response for algebra"
        assert criteria_result == "Expected answer is x=5. Context: solving equations"


class TestGEvalInterpolationHelpers:
    """Tests for GEval template interpolation helper functions."""

    def test_interpolate_template_single_field(self):
        """Test interpolating a single field."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "Expected: {{answer}}"
        dataset_item = {"answer": "42"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "Expected: 42"

    def test_interpolate_template_multiple_fields(self):
        """Test interpolating multiple fields."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "Question: {{question}}, Expected: {{answer}}"
        dataset_item = {"question": "What is 6*7?", "answer": "42"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "Question: What is 6*7?, Expected: 42"

    def test_interpolate_template_missing_field_unchanged(self):
        """Test that missing fields leave placeholder unchanged."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "Expected: {{missing_field}}"
        dataset_item = {"answer": "42"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "Expected: {{missing_field}}"

    def test_interpolate_template_no_placeholders(self):
        """Test template without placeholders returns unchanged."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "No placeholders here"
        dataset_item = {"answer": "42"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "No placeholders here"

    def test_interpolate_template_non_string_values(self):
        """Test that non-string values are converted to strings."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "Count: {{count}}, Active: {{active}}"
        dataset_item = {"count": 123, "active": True}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "Count: 123, Active: True"

    def test_interpolate_template_dotted_keys(self):
        """Test interpolating keys with dots (e.g., user.name)."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "User: {{user.name}}, ID: {{user.id}}"
        dataset_item = {"user.name": "Alice", "user.id": "12345"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "User: Alice, ID: 12345"

    def test_interpolate_template_hyphenated_keys(self):
        """Test interpolating keys with hyphens (e.g., answer-key)."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "Answer: {{answer-key}}, Type: {{response-type}}"
        dataset_item = {"answer-key": "correct", "response-type": "multiple-choice"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "Answer: correct, Type: multiple-choice"

    def test_interpolate_template_mixed_special_chars(self):
        """Test interpolating keys with mixed dots, hyphens, and underscores."""
        from opik_backend.studio.metrics import _interpolate_template
        
        template = "Value: {{var_with-special.chars}}"
        dataset_item = {"var_with-special.chars": "complex_value"}
        result = _interpolate_template(template, dataset_item)
        
        assert result == "Value: complex_value"

    def test_has_template_placeholders_true(self):
        """Test detecting placeholders in text."""
        from opik_backend.studio.metrics import _has_template_placeholders
        
        assert _has_template_placeholders("Contains {{field}}") is True
        assert _has_template_placeholders("Multiple {{a}} and {{b}}") is True

    def test_has_template_placeholders_false(self):
        """Test detecting no placeholders in text."""
        from opik_backend.studio.metrics import _has_template_placeholders
        
        assert _has_template_placeholders("No placeholders") is False
        assert _has_template_placeholders("Single braces {field}") is False
        assert _has_template_placeholders("") is False


class TestTemplateSyntaxConversion:
    """Tests for template syntax conversion from {{var}} to {var}."""

    def test_convert_single_variable(self):
        """Test converting single variable."""
        result = _convert_template_syntax("Hello {{name}}")
        assert result == "Hello {name}"

    def test_convert_multiple_variables(self):
        """Test converting multiple variables."""
        result = _convert_template_syntax("{{greeting}} {{name}}!")
        assert result == "{greeting} {name}!"

    def test_preserve_single_braces(self):
        """Test that single braces are preserved."""
        result = _convert_template_syntax("Already {converted}")
        assert result == "Already {converted}"

    def test_no_variables(self):
        """Test string without variables."""
        result = _convert_template_syntax("No variables here")
        assert result == "No variables here"

    def test_empty_string(self):
        """Test empty string."""
        result = _convert_template_syntax("")
        assert result == ""

    def test_variable_in_sentence(self):
        """Test variable embedded in sentence."""
        result = _convert_template_syntax("What is the mime type for {{url}}?")
        assert result == "What is the mime type for {url}?"

    def test_optimization_config_converts_templates(self):
        """Test OptimizationConfig.from_dict converts template syntax in prompt messages."""
        config = {
            "dataset_name": "test_dataset",
            "prompt": {
                "messages": [
                    {"role": "system", "content": "Be helpful"},
                    {"role": "user", "content": "What is {{question}}? Answer: {{answer}}"}
                ]
            },
            "llm_model": {"model": "gpt-4o-mini", "parameters": {}},
            "evaluation": {"metrics": [{"type": "equals", "parameters": {}}]},
            "optimizer": {"type": "gepa", "parameters": {}}
        }
        
        opt_config = OptimizationConfig.from_dict(config)
        
        # System message should be unchanged (no variables)
        assert opt_config.prompt_messages[0]["content"] == "Be helpful"
        # User message should have converted variables
        assert opt_config.prompt_messages[1]["content"] == "What is {question}? Answer: {answer}"