wehub-resource-sync
102 lines
3.7 KiB
Python
102 lines
3.7 KiB
Python
import os
|
|
import pytest
|
|
from deepeval.metrics import ImageReferenceMetric
|
|
from deepeval.test_case import LLMTestCase, MLLMImage, ToolCall
|
|
from deepeval import evaluate
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
os.getenv("OPENAI_API_KEY") is None
|
|
or not os.getenv("OPENAI_API_KEY").strip(),
|
|
reason="OPENAI_API_KEY is not set",
|
|
)
|
|
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
CAR = os.path.join(current_dir, "images/car.png")
|
|
|
|
|
|
class TestImageReferenceMetric:
|
|
"""Tests for answer relevancy metric"""
|
|
|
|
def test_multimodal_async_metric_measure(self):
|
|
image = MLLMImage(url=CAR)
|
|
test_case = LLMTestCase(
|
|
input=f"What's shown in this image? {image}'",
|
|
expected_output=f"That's an image of a car",
|
|
actual_output=f"That is a car {image}.",
|
|
retrieval_context=[f"Cars are great to look at {image}"],
|
|
context=[f"Cars are great to look at {image}"],
|
|
tools_called=[
|
|
ToolCall(name="ImageAnalysis"),
|
|
ToolCall(name="ToolQuery"),
|
|
],
|
|
expected_tools=[ToolCall(name="ImageAnalysis")],
|
|
)
|
|
metric = ImageReferenceMetric()
|
|
metric.measure(test_case)
|
|
|
|
assert metric.score is not None
|
|
assert metric.reason is not None
|
|
assert test_case.multimodal is True
|
|
|
|
def test_multimodal_sync_metric_measure(self):
|
|
image = MLLMImage(url=CAR)
|
|
test_case = LLMTestCase(
|
|
input=f"What's shown in this image? {image}'",
|
|
expected_output=f"That's an image of a car",
|
|
actual_output=f"That is a car {image}.",
|
|
retrieval_context=[f"Cars are great to look at {image}"],
|
|
context=[f"Cars are great to look at {image}"],
|
|
tools_called=[
|
|
ToolCall(name="ImageAnalysis"),
|
|
ToolCall(name="ToolQuery"),
|
|
],
|
|
expected_tools=[ToolCall(name="ImageAnalysis")],
|
|
)
|
|
metric = ImageReferenceMetric(async_mode=False)
|
|
metric.measure(test_case)
|
|
|
|
assert metric.score is not None
|
|
assert metric.reason is not None
|
|
assert test_case.multimodal is True
|
|
|
|
def test_invalid_model_throws_error_for_multimodal(self):
|
|
image = MLLMImage(url=CAR)
|
|
test_case = LLMTestCase(
|
|
input=f"What's shown in this image? {image}'",
|
|
expected_output=f"That's an image of a car",
|
|
actual_output=f"That is a car {image}.",
|
|
retrieval_context=[f"Cars are great to look at {image}"],
|
|
context=[f"Cars are great to look at {image}"],
|
|
tools_called=[
|
|
ToolCall(name="ImageAnalysis"),
|
|
ToolCall(name="ToolQuery"),
|
|
],
|
|
expected_tools=[ToolCall(name="ImageAnalysis")],
|
|
)
|
|
with pytest.raises(ValueError):
|
|
metric = ImageReferenceMetric(
|
|
async_mode=False, model="gpt-3.5-turbo"
|
|
)
|
|
metric.measure(test_case)
|
|
|
|
def test_multimodal_evaluate_method(self):
|
|
image = MLLMImage(url=CAR)
|
|
test_case = LLMTestCase(
|
|
input=f"What's shown in this image? {image}'",
|
|
expected_output=f"That's an image of a car",
|
|
actual_output=f"That is a car {image}.",
|
|
retrieval_context=[f"Cars are great to look at {image}"],
|
|
context=[f"Cars are great to look at {image}"],
|
|
tools_called=[
|
|
ToolCall(name="ImageAnalysis"),
|
|
ToolCall(name="ToolQuery"),
|
|
],
|
|
expected_tools=[ToolCall(name="ImageAnalysis")],
|
|
)
|
|
|
|
metric = ImageReferenceMetric()
|
|
|
|
results = evaluate([test_case], [metric])
|
|
|
|
assert results is not None
|