567-labs--instructor/tests/v2/test_client_unified.py

"""Unified parametrized tests for all provider client factories.

These tests verify client factory behavior (mode normalization, registry, errors, imports)
across all providers without requiring API keys.
"""

from __future__ import annotations

import importlib.util
from pathlib import Path
from typing import Any

import pytest

from instructor import Mode, Provider
from instructor.v2.core.registry import mode_registry, normalize_mode
from tests.v2.provider_matrix import legacy_config_dicts

_PROJECT_ROOT = Path(__file__).resolve().parents[2]
_HANDLER_MODULE_PATHS: dict[Provider, Path] = {
    Provider.OPENAI: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.ANYSCALE: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.TOGETHER: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.DATABRICKS: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.DEEPSEEK: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.ANTHROPIC: _PROJECT_ROOT / "instructor/v2/providers/anthropic/handlers.py",
    Provider.GENAI: _PROJECT_ROOT / "instructor/v2/providers/genai/handlers.py",
    Provider.GEMINI: _PROJECT_ROOT / "instructor/v2/providers/gemini/handlers.py",
    Provider.COHERE: _PROJECT_ROOT / "instructor/v2/providers/cohere/handlers.py",
    Provider.OPENROUTER: _PROJECT_ROOT
    / "instructor/v2/providers/openrouter/handlers.py",
    Provider.PERPLEXITY: _PROJECT_ROOT
    / "instructor/v2/providers/perplexity/handlers.py",
    Provider.XAI: _PROJECT_ROOT / "instructor/v2/providers/xai/handlers.py",
    Provider.GROQ: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.MISTRAL: _PROJECT_ROOT / "instructor/v2/providers/mistral/handlers.py",
    Provider.FIREWORKS: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.CEREBRAS: _PROJECT_ROOT / "instructor/v2/providers/openai/handlers.py",
    Provider.WRITER: _PROJECT_ROOT / "instructor/v2/providers/writer/handlers.py",
    Provider.BEDROCK: _PROJECT_ROOT / "instructor/v2/providers/bedrock/handlers.py",
    Provider.VERTEXAI: _PROJECT_ROOT / "instructor/v2/providers/vertexai/handlers.py",
}
_HANDLERS_LOADED: set[Provider] = set()


def _clear_proxy_env(monkeypatch: pytest.MonkeyPatch) -> None:
    for key in (
        "ALL_PROXY",
        "all_proxy",
        "HTTPS_PROXY",
        "https_proxy",
        "HTTP_PROXY",
        "http_proxy",
    ):
        monkeypatch.delenv(key, raising=False)


def _ensure_handlers_loaded(provider: Provider) -> None:
    if provider in _HANDLERS_LOADED:
        return
    provider_modes = PROVIDER_CLIENT_CONFIGS.get(provider, {}).get(
        "supported_modes", []
    )
    if provider_modes and all(
        mode_registry.is_registered(provider, mode) for mode in provider_modes
    ):
        _HANDLERS_LOADED.add(provider)
        return
    handler_path = _HANDLER_MODULE_PATHS.get(provider)
    if handler_path is None or not handler_path.exists():
        return
    spec = importlib.util.spec_from_file_location(
        f"tests.v2.handlers_{provider.value}",
        handler_path,
    )
    if spec is None or spec.loader is None:
        return
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    _HANDLERS_LOADED.add(provider)


PROVIDER_CLIENT_CONFIGS: dict[Provider, dict[str, Any]] = legacy_config_dicts()


def _dependency_missing(module: str) -> bool:
    """Check if a dependency module is missing."""
    try:
        return importlib.util.find_spec(module.split(".")[0]) is None
    except ModuleNotFoundError:
        return True


def _is_expected_missing_dependency(provider: Provider, exc: ImportError) -> bool:
    """Return True when an import failed because the provider SDK is unavailable."""
    sdk_module = PROVIDER_CLIENT_CONFIGS[provider]["sdk_module"]
    expected_root = str(sdk_module).split(".")[0]
    missing_name = getattr(exc, "name", None)
    if missing_name:
        return missing_name.split(".")[0] == expected_root

    return f"No module named '{expected_root}'" in str(exc)


def _get_provider_params():
    """Generate provider parameters for parametrized tests."""
    return [
        pytest.param(provider, id=provider.value)
        for provider in PROVIDER_CLIENT_CONFIGS.keys()
    ]


def _get_provider_mode_params():
    """Generate (provider, mode) parameters for supported modes."""
    params = []
    for provider, config in PROVIDER_CLIENT_CONFIGS.items():
        for mode in config["supported_modes"]:
            params.append(
                pytest.param(provider, mode, id=f"{provider.value}-{mode.value}")
            )
    return params


def _get_provider_unsupported_mode_params():
    """Generate (provider, mode) parameters for unsupported modes."""
    params = []
    for provider, config in PROVIDER_CLIENT_CONFIGS.items():
        for mode in config["unsupported_modes"]:
            params.append(
                pytest.param(provider, mode, id=f"{provider.value}-{mode.value}")
            )
    return params


def _get_provider_legacy_mode_params():
    """Generate (provider, legacy_mode) parameters."""
    params = []
    for provider, config in PROVIDER_CLIENT_CONFIGS.items():
        for legacy_mode in config["legacy_modes"].keys():
            params.append(
                pytest.param(
                    provider,
                    legacy_mode,
                    id=f"{provider.value}-{legacy_mode.value}",
                )
            )
    return params


# ============================================================================
# Mode Registry Tests
# ============================================================================


@pytest.mark.parametrize("provider,mode", _get_provider_mode_params())
def test_supported_mode_is_registered(provider: Provider, mode: Mode) -> None:
    """Test that all supported modes are registered in the registry."""
    _ensure_handlers_loaded(provider)
    assert mode_registry.is_registered(provider, mode), (
        f"Mode {mode.value} should be registered for {provider.value}"
    )


@pytest.mark.parametrize("provider,mode", _get_provider_unsupported_mode_params())
def test_unsupported_mode_not_registered(provider: Provider, mode: Mode) -> None:
    """Test that unsupported modes are NOT registered."""
    assert not mode_registry.is_registered(provider, mode), (
        f"Mode {mode.value} should NOT be registered for {provider.value}"
    )


@pytest.mark.parametrize("provider", _get_provider_params())
def test_get_modes_for_provider(provider: Provider) -> None:
    """Test getting all modes for a provider."""
    _ensure_handlers_loaded(provider)
    config = PROVIDER_CLIENT_CONFIGS[provider]
    registered_modes = mode_registry.get_modes_for_provider(provider)

    # All supported modes should be registered
    for mode in config["supported_modes"]:
        assert mode in registered_modes, (
            f"Mode {mode.value} should be in registered modes for {provider.value}"
        )

    # Unsupported modes should not be registered
    for mode in config["unsupported_modes"]:
        assert mode not in registered_modes, (
            f"Mode {mode.value} should NOT be in registered modes for {provider.value}"
        )


@pytest.mark.parametrize("provider,mode", _get_provider_mode_params())
def test_handlers_have_all_methods(provider: Provider, mode: Mode) -> None:
    """Test that all handlers have required methods."""
    _ensure_handlers_loaded(provider)
    handlers = mode_registry.get_handlers(provider, mode)

    assert handlers.request_handler is not None
    assert handlers.reask_handler is not None
    assert handlers.response_parser is not None


# ============================================================================
# Mode Normalization Tests
# ============================================================================


@pytest.mark.parametrize("provider,mode", _get_provider_mode_params())
def test_generic_mode_passes_through(provider: Provider, mode: Mode) -> None:
    """Test that generic modes pass through unchanged."""
    result = normalize_mode(provider, mode)
    assert result == mode, (
        f"Generic mode {mode.value} should pass through unchanged for {provider.value}"
    )


@pytest.mark.parametrize("provider,legacy_mode", _get_provider_legacy_mode_params())
def test_legacy_mode_normalizes_to_registered_mode(
    provider: Provider, legacy_mode: Mode
) -> None:
    """Legacy provider-specific modes normalize to registered v2 modes."""
    result = normalize_mode(provider, legacy_mode)
    assert result != legacy_mode
    assert mode_registry.is_registered(provider, legacy_mode), (
        f"Legacy mode {legacy_mode.value} should remain accepted for {provider.value}"
    )


# ============================================================================
# Import Tests
# ============================================================================


@pytest.mark.parametrize("provider", _get_provider_params())
def test_from_function_importable(provider: Provider) -> None:
    """Test that from_* function is importable from instructor.v2."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    from_function = config["from_function"]

    # Import from instructor.v2
    module = __import__("instructor.v2", fromlist=[from_function])
    func = getattr(module, from_function, None)

    # Should be None if SDK not installed, or a callable if installed
    assert func is None or callable(func), (
        f"{from_function} should be None or callable, got {type(func)}"
    )


@pytest.mark.parametrize("provider", _get_provider_params())
def test_handlers_importable(provider: Provider) -> None:
    """Test that handlers are importable."""
    handler_path = _HANDLER_MODULE_PATHS.get(provider)
    assert handler_path is not None and handler_path.exists(), (
        f"Missing handler module path for {provider.value}"
    )

    _ensure_handlers_loaded(provider)

    assert any(
        mode_registry.is_registered(provider, mode)
        for mode in PROVIDER_CLIENT_CONFIGS[provider]["supported_modes"]
    ), f"No registered handlers found for {provider.value}"


# ============================================================================
# Error Handling Tests
# ============================================================================


@pytest.mark.parametrize("provider,mode", _get_provider_unsupported_mode_params())
def test_unsupported_mode_raises_error(provider: Provider, mode: Mode) -> None:
    """Test that getting handlers for unsupported mode raises KeyError."""
    with pytest.raises(KeyError):
        mode_registry.get_handlers(provider, mode)


@pytest.mark.parametrize("provider", _get_provider_params())
def test_parallel_tools_not_supported_unless_registered(provider: Provider) -> None:
    """Test that PARALLEL_TOOLS is not supported unless registered."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    is_supported = Mode.PARALLEL_TOOLS in config["supported_modes"]
    is_registered = mode_registry.is_registered(provider, Mode.PARALLEL_TOOLS)

    assert is_supported == is_registered, (
        f"PARALLEL_TOOLS support mismatch for {provider.value}: "
        f"supported={is_supported}, registered={is_registered}"
    )


@pytest.mark.parametrize("provider", _get_provider_params())
def test_responses_tools_not_supported_unless_registered(provider: Provider) -> None:
    """Test that RESPONSES_TOOLS is not supported unless registered."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    is_supported = Mode.RESPONSES_TOOLS in config["supported_modes"]
    is_registered = mode_registry.is_registered(provider, Mode.RESPONSES_TOOLS)

    assert is_supported == is_registered, (
        f"RESPONSES_TOOLS support mismatch for {provider.value}: "
        f"supported={is_supported}, registered={is_registered}"
    )


# ============================================================================
# SDK Availability Tests
# ============================================================================


@pytest.mark.parametrize("provider", _get_provider_params())
def test_from_function_raises_without_sdk(provider: Provider) -> None:
    """Test that from_* function raises error when SDK not installed."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    sdk_module = config["sdk_module"]
    from_function = config["from_function"]

    if not _dependency_missing(sdk_module):
        pytest.skip(
            f"{sdk_module} is installed"  # ty: ignore[too-many-positional-arguments]
        )

    # Try to import the from_* function from the provider's client module
    try:
        client_module_path = f"instructor.v2.providers.{provider.value}.client"
        client_module = __import__(client_module_path, fromlist=[from_function])
        from_function_obj = getattr(client_module, from_function, None)

        if from_function_obj is None:
            pytest.skip(
                f"{from_function} not found in client module"  # ty: ignore[too-many-positional-arguments]
            )

        from instructor.core.exceptions import ClientError

        expected_message = config.get(
            "missing_sdk_message",
            f"{sdk_module.split('.')[0]} is not installed",
        )
        with pytest.raises(ClientError, match=expected_message):
            from_function_obj("not a client")  # type: ignore[call-arg]
    except (ImportError, ModuleNotFoundError) as exc:
        if _is_expected_missing_dependency(provider, exc):
            pytest.skip(
                f"{sdk_module} import path is unavailable in this environment"  # ty: ignore[too-many-positional-arguments]
            )
        raise


# ============================================================================
# String-Based Initialization Tests
# ============================================================================


# OpenAI-compatible providers that support string-based initialization
_OPENAI_COMPAT_PROVIDERS = [
    Provider.ANYSCALE,
    Provider.TOGETHER,
    Provider.DATABRICKS,
    Provider.DEEPSEEK,
]


@pytest.mark.parametrize(
    "provider",
    [pytest.param(p, id=p.value) for p in _OPENAI_COMPAT_PROVIDERS],
)
def test_string_based_initialization_delegates_to_from_provider(
    provider: Provider,
) -> None:
    """Test that string-based initialization delegates to from_provider."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    from_function = config["from_function"]

    # Import the from_* function
    module = __import__("instructor.v2", fromlist=[from_function])
    func = getattr(module, from_function, None)

    if func is None:
        pytest.skip(
            f"{from_function} not available (SDK may not be installed)"  # ty: ignore[too-many-positional-arguments]
        )

    # Mock from_provider to verify it's called
    from unittest.mock import patch

    with patch("instructor.from_provider") as mock_from_provider:
        # Call with string (model name)
        func("test-model", mode=Mode.TOOLS)

        # Verify from_provider was called with correct provider prefix
        mock_from_provider.assert_called_once()
        call_args = mock_from_provider.call_args
        assert call_args[0][0] == f"{provider.value}/test-model"
        assert call_args[1]["mode"] == Mode.TOOLS


@pytest.mark.parametrize(
    "provider",
    [pytest.param(p, id=p.value) for p in _OPENAI_COMPAT_PROVIDERS],
)
def test_string_based_initialization_with_async_client(provider: Provider) -> None:
    """Test that string-based initialization supports async_client parameter."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    from_function = config["from_function"]

    # Import the from_* function
    module = __import__("instructor.v2", fromlist=[from_function])
    func = getattr(module, from_function, None)

    if func is None:
        pytest.skip(
            f"{from_function} not available (SDK may not be installed)"  # ty: ignore[too-many-positional-arguments]
        )

    # Mock from_provider to verify it's called
    from unittest.mock import patch

    with patch("instructor.from_provider") as mock_from_provider:
        # Call with string and async_client=True
        func("test-model", mode=Mode.TOOLS, async_client=True)

        # Verify from_provider was called with async_client=True
        mock_from_provider.assert_called_once()
        call_args = mock_from_provider.call_args
        assert call_args[0][0] == f"{provider.value}/test-model"
        assert call_args[1]["mode"] == Mode.TOOLS
        assert call_args[1]["async_client"] is True


@pytest.mark.parametrize(
    "provider",
    [pytest.param(p, id=p.value) for p in _OPENAI_COMPAT_PROVIDERS],
)
def test_string_based_initialization_forwards_kwargs(provider: Provider) -> None:
    """Test that string-based initialization forwards all kwargs to from_provider."""
    config = PROVIDER_CLIENT_CONFIGS[provider]
    from_function = config["from_function"]

    # Import the from_* function
    module = __import__("instructor.v2", fromlist=[from_function])
    func = getattr(module, from_function, None)

    if func is None:
        pytest.skip(
            f"{from_function} not available (SDK may not be installed)"  # ty: ignore[too-many-positional-arguments]
        )

    # Mock from_provider to verify it's called
    from unittest.mock import patch

    with patch("instructor.from_provider") as mock_from_provider:
        # Call with string and additional kwargs
        func(
            "test-model",
            mode=Mode.TOOLS,
            api_key="test-key",
            base_url="https://test.example.com",
            timeout=30,
        )

        # Verify from_provider was called with all kwargs
        mock_from_provider.assert_called_once()
        call_args = mock_from_provider.call_args
        assert call_args[0][0] == f"{provider.value}/test-model"
        assert call_args[1]["mode"] == Mode.TOOLS
        assert call_args[1]["api_key"] == "test-key"
        assert call_args[1]["base_url"] == "https://test.example.com"
        assert call_args[1]["timeout"] == 30


@pytest.mark.parametrize(
    "provider",
    [pytest.param(p, id=p.value) for p in _OPENAI_COMPAT_PROVIDERS],
)
def test_client_based_initialization_still_works(
    provider: Provider, monkeypatch: pytest.MonkeyPatch
) -> None:
    """Test that client-based initialization still works (backward compatibility)."""
    from unittest.mock import patch

    config = PROVIDER_CLIENT_CONFIGS[provider]
    from_function = config["from_function"]
    sdk_module = config["sdk_module"]

    # Skip if SDK not installed
    if _dependency_missing(sdk_module):
        pytest.skip(
            f"{sdk_module} not installed"  # ty: ignore[too-many-positional-arguments]
        )

    # Import the from_* function
    module = __import__("instructor.v2", fromlist=[from_function])
    func = getattr(module, from_function, None)

    if func is None:
        pytest.skip(
            f"{from_function} not available"  # ty: ignore[too-many-positional-arguments]
        )

    # Import OpenAI client
    try:
        import openai
    except ImportError:
        pytest.skip(
            "openai package not installed"  # ty: ignore[too-many-positional-arguments]
        )

    _clear_proxy_env(monkeypatch)

    # Create a mock OpenAI client
    client = openai.OpenAI(api_key="test-key")

    # Call with client (should use _from_openai_compat, not from_provider)
    with patch(
        "instructor.v2.providers.openai.client._from_openai_compat"
    ) as mock_compat:
        mock_compat.return_value = "mock_instructor"
        result = func(client, mode=Mode.TOOLS)

        # Verify _from_openai_compat was called (not from_provider)
        mock_compat.assert_called_once()
        call_args = mock_compat.call_args
        assert call_args[0][0] == client
        assert call_args[1]["provider"] == provider
        assert call_args[1]["mode"] == Mode.TOOLS