learningcircuit--local-deep-research/tests/llm_providers/test_custom_endpoint_ci_integration.py

"""
CI Integration tests for OpenAI-Compatible Endpoint.

These tests use real API calls to verify the custom endpoint URL handling works.
They require OPENROUTER_API_KEY to be set in the environment (available in CI).

REGRESSION TEST: This test would have caught the v1.3.10+ regression where
custom endpoint URLs were not being passed correctly to the OpenAI client.
"""

import os
import pytest

from local_deep_research.llm.providers.implementations.custom_openai_endpoint import (
    CustomOpenAIEndpointProvider,
)


# Mark for tests that require the API key
requires_openrouter_key = pytest.mark.skipif(
    not os.environ.get("OPENROUTER_API_KEY"),
    reason="OPENROUTER_API_KEY not set - skipping CI integration tests",
)


@requires_openrouter_key
class TestCustomEndpointWithOpenRouter:
    """Integration tests using OpenRouter as a real OpenAI-compatible endpoint.

    These tests verify that CustomOpenAIEndpointProvider correctly handles
    custom URLs by actually connecting to OpenRouter's API.
    """

    OPENROUTER_URL = "https://openrouter.ai/api/v1"

    def test_fetch_models_from_openrouter(self):
        """Can fetch models from OpenRouter using custom endpoint.

        REGRESSION TEST: This is the exact scenario that broke in v1.3.10+.
        The custom URL was not being passed to the OpenAI client.
        """
        api_key = os.environ.get("OPENROUTER_API_KEY")

        # Use CustomOpenAIEndpointProvider with OpenRouter URL
        models = CustomOpenAIEndpointProvider.list_models_for_api(
            api_key=api_key,
            base_url=self.OPENROUTER_URL,
        )

        # OpenRouter should return many models
        assert isinstance(models, list)
        assert len(models) > 0, (
            "Should fetch at least one model from OpenRouter"
        )

        # Verify model format
        for model in models[:5]:  # Check first 5
            assert "value" in model
            assert "label" in model
            assert isinstance(model["value"], str)
            assert len(model["value"]) > 0

    def test_models_include_known_providers(self):
        """OpenRouter models include known providers like OpenAI, Anthropic.

        This verifies we're actually getting real model data.
        """
        api_key = os.environ.get("OPENROUTER_API_KEY")

        models = CustomOpenAIEndpointProvider.list_models_for_api(
            api_key=api_key,
            base_url=self.OPENROUTER_URL,
        )

        model_ids = [m["value"] for m in models]

        # OpenRouter should have models from major providers
        # Check for at least one of these patterns
        has_known_model = any(
            any(
                provider in model_id
                for provider in ["openai", "anthropic", "google", "meta"]
            )
            for model_id in model_ids
        )

        assert has_known_model, (
            f"Expected models from known providers, got: {model_ids[:10]}"
        )

    def test_url_not_modified(self):
        """The URL we pass is used as-is (no unwanted /v1 suffix added).

        REGRESSION TEST: Previously the code was adding /v1 to URLs,
        which would break URLs that already have it.
        """
        api_key = os.environ.get("OPENROUTER_API_KEY")

        # URL already has /v1 - should work without modification
        url_with_v1 = "https://openrouter.ai/api/v1"

        models = CustomOpenAIEndpointProvider.list_models_for_api(
            api_key=api_key,
            base_url=url_with_v1,
        )

        # If URL was modified to /v1/v1, this would fail
        assert len(models) > 0, "URL should work as-is without modification"


class TestCustomEndpointAPIContract:
    """Tests verifying the API contract is correct for CI usage."""

    def test_base_url_parameter_exists(self):
        """list_models_for_api has base_url as second parameter.

        This ensures the API contract used by settings_routes.py is stable.
        """
        import inspect

        sig = inspect.signature(
            CustomOpenAIEndpointProvider.list_models_for_api
        )
        params = list(sig.parameters.keys())

        assert params == ["api_key", "base_url"], (
            f"Expected ['api_key', 'base_url'], got {params}"
        )

    @requires_openrouter_key
    def test_works_without_settings_snapshot(self):
        """Can call list_models_for_api without any settings infrastructure.

        REGRESSION TEST: The v1.3.10+ code required settings_snapshot which
        added unnecessary complexity and broke the feature.
        """
        api_key = os.environ.get("OPENROUTER_API_KEY")

        # Should work with just api_key and base_url - no settings needed
        models = CustomOpenAIEndpointProvider.list_models_for_api(
            api_key=api_key,
            base_url="https://openrouter.ai/api/v1",
        )

        assert len(models) > 0