wehub-resource-sync
279 lines
9.4 KiB
Python
279 lines
9.4 KiB
Python
"""Tests for image_to_text tool and multimodal detection."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openharness.api.provider import is_model_multimodal
|
|
from openharness.config.settings import VisionModelConfig
|
|
from openharness.tools.base import ToolExecutionContext
|
|
from openharness.tools.image_to_text_tool import ImageToTextTool, ImageToTextToolInput
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# is_model_multimodal tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.mark.parametrize(
|
|
("model", "expected"),
|
|
[
|
|
# Anthropic Claude 3+ (multimodal)
|
|
("claude-sonnet-4-6", True),
|
|
("claude-opus-4-6", True),
|
|
("claude-haiku-4-5", True),
|
|
("claude-3-5-sonnet-20241022", True),
|
|
("claude-3-opus-20240229", True),
|
|
("claude-3-haiku-20240307", True),
|
|
# OpenAI multimodal
|
|
("gpt-4o", True),
|
|
("gpt-4o-mini", True),
|
|
("o1-mini", True),
|
|
("o3-mini", True),
|
|
("o4-mini", True),
|
|
# Google Gemini
|
|
("gemini-2.5-flash", True),
|
|
("gemini-2.0-flash", True),
|
|
("gemini-pro-vision", True),
|
|
# Qwen VL
|
|
("qwen-vl-max", True),
|
|
("qwen2.5-vl-72b", True),
|
|
("qvq-72b-preview", True),
|
|
# DeepSeek VL
|
|
("deepseek-vl2", True),
|
|
# Other multimodal
|
|
("llava-v1.6-34b", True),
|
|
("pixtral-12b", True),
|
|
("step-2-16k", True),
|
|
("step-1v-32k", True),
|
|
("kimi-k2.5", True),
|
|
# Non-multimodal models
|
|
("claude-2.1", False),
|
|
("gpt-4", False),
|
|
("gpt-3.5-turbo", False),
|
|
("deepseek-chat", False),
|
|
("deepseek-reasoner", False),
|
|
("qwen-turbo", False),
|
|
("qwen-plus", False),
|
|
("kimi-k2", False),
|
|
("step-1-8k", False),
|
|
("glm-4", False),
|
|
("gemini-1.0-pro", False),
|
|
("unknown-model-123", False),
|
|
("", False),
|
|
# With provider prefix
|
|
("anthropic/claude-sonnet-4-6", True),
|
|
("openai/gpt-4o", True),
|
|
("openai/gpt-4", False),
|
|
],
|
|
)
|
|
def test_is_model_multimodal(model: str, expected: bool) -> None:
|
|
assert is_model_multimodal(model) == expected
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ImageToTextTool input validation tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestImageToTextToolInput:
|
|
"""Validate the tool's input model."""
|
|
|
|
def test_valid_image_data(self) -> None:
|
|
inp = ImageToTextToolInput(
|
|
image_data="iVBORw0KGgo=",
|
|
media_type="image/png",
|
|
)
|
|
assert inp.image_data == "iVBORw0KGgo="
|
|
assert inp.media_type == "image/png"
|
|
assert inp.prompt # default prompt
|
|
|
|
def test_valid_image_path(self) -> None:
|
|
inp = ImageToTextToolInput(
|
|
image_path="/tmp/test.png",
|
|
)
|
|
assert inp.image_path == "/tmp/test.png"
|
|
assert inp.image_data is None
|
|
|
|
def test_default_prompt(self) -> None:
|
|
inp = ImageToTextToolInput(image_data="data")
|
|
assert "image" in inp.prompt.lower()
|
|
|
|
def test_custom_prompt(self) -> None:
|
|
inp = ImageToTextToolInput(
|
|
image_data="data",
|
|
prompt="Extract all text from this image",
|
|
)
|
|
assert inp.prompt == "Extract all text from this image"
|
|
|
|
def test_max_tokens_range(self) -> None:
|
|
# Default
|
|
inp = ImageToTextToolInput(image_data="data")
|
|
assert inp.max_tokens == 2048
|
|
|
|
# Custom valid
|
|
inp = ImageToTextToolInput(image_data="data", max_tokens=4096)
|
|
assert inp.max_tokens == 4096
|
|
|
|
def test_neither_image_data_nor_path(self) -> None:
|
|
"""Both fields are optional in the model, but the tool will error."""
|
|
inp = ImageToTextToolInput()
|
|
assert inp.image_data is None
|
|
assert inp.image_path is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ImageToTextTool execution tests (no real API calls)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_no_input(tmp_path: Path) -> None:
|
|
"""Tool returns error when neither image_data nor image_path is provided."""
|
|
tool = ImageToTextTool()
|
|
context = ToolExecutionContext(cwd=tmp_path)
|
|
result = await tool.execute(
|
|
ImageToTextToolInput(),
|
|
context,
|
|
)
|
|
assert result.is_error
|
|
assert "provide either" in result.output
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_nonexistent_path(tmp_path: Path) -> None:
|
|
"""Tool returns error when image_path does not exist."""
|
|
tool = ImageToTextTool()
|
|
context = ToolExecutionContext(cwd=tmp_path)
|
|
result = await tool.execute(
|
|
ImageToTextToolInput(image_path="/nonexistent/path/image.png"),
|
|
context,
|
|
)
|
|
assert result.is_error
|
|
assert "provide either" in result.output
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_no_vision_config(tmp_path: Path) -> None:
|
|
"""Tool returns error when vision model is not configured."""
|
|
tool = ImageToTextTool()
|
|
context = ToolExecutionContext(
|
|
cwd=tmp_path,
|
|
metadata={"vision_model_config": {}},
|
|
)
|
|
result = await tool.execute(
|
|
ImageToTextToolInput(image_data="iVBORw0KGgo="),
|
|
context,
|
|
)
|
|
assert result.is_error
|
|
assert "vision model is not configured" in result.output
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_with_image_path(tmp_path: Path) -> None:
|
|
"""Tool reads a real image file and attempts to describe it."""
|
|
# Create a minimal valid PNG file
|
|
png_path = tmp_path / "test_image.png"
|
|
# Minimal valid PNG (1x1 pixel, white)
|
|
minimal_png = bytes([
|
|
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, # PNG signature
|
|
0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, # IHDR chunk
|
|
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
|
|
0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
|
|
0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, # IDAT chunk
|
|
0x54, 0x08, 0xD7, 0x63, 0x60, 0x60, 0x00, 0x00,
|
|
0x00, 0x04, 0x00, 0x01, 0x27, 0x34, 0x27, 0x0A,
|
|
0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, # IEND chunk
|
|
0xAE, 0x42, 0x60, 0x82,
|
|
])
|
|
png_path.write_bytes(minimal_png)
|
|
|
|
tool = ImageToTextTool()
|
|
context = ToolExecutionContext(
|
|
cwd=tmp_path,
|
|
metadata={
|
|
"vision_model_config": {
|
|
"model": "gpt-4o",
|
|
"api_key": "test-key",
|
|
"base_url": "",
|
|
}
|
|
},
|
|
)
|
|
result = await tool.execute(
|
|
ImageToTextToolInput(image_path=str(png_path)),
|
|
context,
|
|
)
|
|
# Should fail at API call (not at file reading), since the API key is fake
|
|
assert result.is_error
|
|
assert "vision model error" in result.output
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_is_read_only() -> None:
|
|
"""image_to_text is a read-only tool."""
|
|
tool = ImageToTextTool()
|
|
assert tool.is_read_only(ImageToTextToolInput(image_data="data"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VisionModelConfig tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestVisionModelConfig:
|
|
"""Validate the VisionModelConfig model."""
|
|
|
|
def test_default_empty(self) -> None:
|
|
cfg = VisionModelConfig()
|
|
assert cfg.model == ""
|
|
assert cfg.api_key == ""
|
|
assert cfg.base_url == ""
|
|
assert not cfg.is_configured
|
|
|
|
def test_configured(self) -> None:
|
|
cfg = VisionModelConfig(
|
|
model="gpt-4o",
|
|
api_key="sk-test",
|
|
base_url="https://api.openai.com/v1",
|
|
)
|
|
assert cfg.is_configured
|
|
assert cfg.model == "gpt-4o"
|
|
assert cfg.api_key == "sk-test"
|
|
|
|
def test_partial_not_configured(self) -> None:
|
|
cfg = VisionModelConfig(model="gpt-4o")
|
|
assert not cfg.is_configured
|
|
|
|
def test_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("OPENHARNESS_VISION_MODEL", "gpt-4o")
|
|
monkeypatch.setenv("OPENHARNESS_VISION_API_KEY", "sk-env-key")
|
|
monkeypatch.setenv("OPENHARNESS_VISION_BASE_URL", "https://api.example.com/v1")
|
|
|
|
cfg = VisionModelConfig.from_env()
|
|
assert cfg.model == "gpt-4o"
|
|
assert cfg.api_key == "sk-env-key"
|
|
assert cfg.base_url == "https://api.example.com/v1"
|
|
assert cfg.is_configured
|
|
|
|
def test_from_env_partial(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("OPENHARNESS_VISION_MODEL", "gpt-4o")
|
|
monkeypatch.delenv("OPENHARNESS_VISION_API_KEY", raising=False)
|
|
|
|
cfg = VisionModelConfig.from_env()
|
|
assert not cfg.is_configured
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool registry integration test
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_tool_registered() -> None:
|
|
"""image_to_text tool is registered in the default registry."""
|
|
from openharness.tools import create_default_tool_registry
|
|
|
|
registry = create_default_tool_registry()
|
|
tool = registry.get("image_to_text")
|
|
assert tool is not None
|
|
assert tool.name == "image_to_text"
|
|
assert "vision" in tool.description.lower()
|
|
assert tool.input_model.__name__ == "ImageToTextToolInput"
|
|
assert tool.input_model.__module__ == "openharness.tools.image_to_text_tool"
|