wehub-resource-sync
461 lines
17 KiB
Python
461 lines
17 KiB
Python
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
|
|
@pytest.fixture
|
|
def client():
|
|
"""Create test client after environment variables have been cleared by conftest."""
|
|
from api.main import app
|
|
|
|
return TestClient(app)
|
|
|
|
|
|
class TestModelCreation:
|
|
"""Test suite for Model Creation endpoint."""
|
|
|
|
@pytest.mark.asyncio
|
|
@patch("open_notebook.database.repository.repo_query")
|
|
@patch("api.routers.models.Model.save")
|
|
async def test_create_duplicate_model_same_case(
|
|
self, mock_save, mock_repo_query, client
|
|
):
|
|
"""Test that creating a duplicate model with same case returns 400."""
|
|
# Mock repo_query to return a duplicate model
|
|
mock_repo_query.return_value = [
|
|
{
|
|
"id": "model:123",
|
|
"name": "gpt-4",
|
|
"provider": "openai",
|
|
"type": "language",
|
|
}
|
|
]
|
|
|
|
# Attempt to create duplicate
|
|
response = client.post(
|
|
"/api/models",
|
|
json={"name": "gpt-4", "provider": "openai", "type": "language"},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert (
|
|
response.json()["detail"]
|
|
== "Model 'gpt-4' already exists for provider 'openai' with type 'language'"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
@patch("open_notebook.database.repository.repo_query")
|
|
@patch("api.routers.models.Model.save")
|
|
async def test_create_duplicate_model_different_case(
|
|
self, mock_save, mock_repo_query, client
|
|
):
|
|
"""Test that creating a duplicate model with different case returns 400."""
|
|
# Mock repo_query to return a duplicate model (case-insensitive match)
|
|
mock_repo_query.return_value = [
|
|
{
|
|
"id": "model:123",
|
|
"name": "gpt-4",
|
|
"provider": "openai",
|
|
"type": "language",
|
|
}
|
|
]
|
|
|
|
# Attempt to create duplicate with different case
|
|
response = client.post(
|
|
"/api/models",
|
|
json={"name": "GPT-4", "provider": "OpenAI", "type": "language"},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert (
|
|
response.json()["detail"]
|
|
== "Model 'GPT-4' already exists for provider 'OpenAI' with type 'language'"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
@patch("open_notebook.database.repository.repo_query")
|
|
async def test_create_same_model_name_different_provider(
|
|
self, mock_repo_query, client
|
|
):
|
|
"""Test that creating a model with same name but different provider is allowed."""
|
|
from open_notebook.ai.models import Model
|
|
|
|
# Mock repo_query to return empty (no duplicate found for different provider)
|
|
mock_repo_query.return_value = []
|
|
|
|
# Patch the save method on the Model class
|
|
with patch.object(Model, "save", new_callable=AsyncMock):
|
|
# Attempt to create same model name with different provider (anthropic)
|
|
response = client.post(
|
|
"/api/models",
|
|
json={"name": "gpt-4", "provider": "anthropic", "type": "language"},
|
|
)
|
|
|
|
# Should succeed because provider is different
|
|
assert response.status_code == 200
|
|
|
|
@pytest.mark.asyncio
|
|
@patch("open_notebook.database.repository.repo_query")
|
|
async def test_create_same_model_name_different_type(self, mock_repo_query, client):
|
|
"""Test that creating a model with same name but different type is allowed."""
|
|
from open_notebook.ai.models import Model
|
|
|
|
# Mock repo_query to return empty (no duplicate found for different type)
|
|
mock_repo_query.return_value = []
|
|
|
|
# Patch the save method on the Model class
|
|
with patch.object(Model, "save", new_callable=AsyncMock):
|
|
# Attempt to create same model name with different type (embedding instead of language)
|
|
response = client.post(
|
|
"/api/models",
|
|
json={"name": "gpt-4", "provider": "openai", "type": "embedding"},
|
|
)
|
|
|
|
# Should succeed because type is different
|
|
assert response.status_code == 200
|
|
|
|
|
|
class TestModelsProviderAvailability:
|
|
"""Test suite for Models Provider Availability endpoint."""
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_generic_env_var_enables_all_modes(self, mock_esperanto, mock_env, client):
|
|
"""Test that OPENAI_COMPATIBLE_BASE_URL enables all 4 modes."""
|
|
|
|
# Mock environment: only generic var is set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL":
|
|
return "http://localhost:1234/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# openai-compatible should be available
|
|
assert "openai_compatible" in data["available"]
|
|
|
|
# Should support all 4 types
|
|
assert "openai_compatible" in data["supported_types"]
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert "language" in supported
|
|
assert "embedding" in supported
|
|
assert "speech_to_text" in supported
|
|
assert "text_to_speech" in supported
|
|
assert len(supported) == 4
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_mode_specific_env_vars_llm_embedding(
|
|
self, mock_esperanto, mock_env, client
|
|
):
|
|
"""Test mode-specific env vars (LLM + EMBEDDING) enable only those 2 modes."""
|
|
|
|
# Mock environment: only LLM and EMBEDDING specific vars are set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_LLM":
|
|
return "http://localhost:1234/v1"
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_EMBEDDING":
|
|
return "http://localhost:8080/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# openai-compatible should be available
|
|
assert "openai_compatible" in data["available"]
|
|
|
|
# Should support only language and embedding
|
|
assert "openai_compatible" in data["supported_types"]
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert "language" in supported
|
|
assert "embedding" in supported
|
|
assert "speech_to_text" not in supported
|
|
assert "text_to_speech" not in supported
|
|
assert len(supported) == 2
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_no_env_vars_set(self, mock_esperanto, mock_env, client):
|
|
"""Test that openai-compatible is not available when no env vars are set."""
|
|
|
|
# Mock environment: no openai-compatible vars are set
|
|
def env_side_effect(key):
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# openai-compatible should NOT be available
|
|
assert "openai_compatible" not in data["available"]
|
|
assert "openai_compatible" in data["unavailable"]
|
|
|
|
# Should not have supported_types entry
|
|
assert "openai_compatible" not in data["supported_types"]
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_mixed_config_generic_and_mode_specific(
|
|
self, mock_esperanto, mock_env, client
|
|
):
|
|
"""Test mixed config: generic + mode-specific (generic should enable all)."""
|
|
|
|
# Mock environment: both generic and mode-specific vars are set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL":
|
|
return "http://localhost:1234/v1"
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_LLM":
|
|
return "http://localhost:5678/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# openai-compatible should be available
|
|
assert "openai_compatible" in data["available"]
|
|
|
|
# Generic var enables all, so all 4 should be supported
|
|
assert "openai_compatible" in data["supported_types"]
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert "language" in supported
|
|
assert "embedding" in supported
|
|
assert "speech_to_text" in supported
|
|
assert "text_to_speech" in supported
|
|
assert len(supported) == 4
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_individual_mode_llm_only(self, mock_esperanto, mock_env, client):
|
|
"""Test individual mode-specific var (LLM only)."""
|
|
|
|
# Mock environment: only LLM specific var is set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_LLM":
|
|
return "http://localhost:1234/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Should support only language
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert supported == ["language"]
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_individual_mode_embedding_only(self, mock_esperanto, mock_env, client):
|
|
"""Test individual mode-specific var (EMBEDDING only)."""
|
|
|
|
# Mock environment: only EMBEDDING specific var is set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_EMBEDDING":
|
|
return "http://localhost:8080/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Should support only embedding
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert supported == ["embedding"]
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_individual_mode_stt_only(self, mock_esperanto, mock_env, client):
|
|
"""Test individual mode-specific var (STT only)."""
|
|
|
|
# Mock environment: only STT specific var is set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_STT":
|
|
return "http://localhost:9000/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Should support only speech_to_text
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert supported == ["speech_to_text"]
|
|
|
|
@patch("api.routers.models.os.environ.get")
|
|
@patch("api.routers.models.AIFactory.get_available_providers")
|
|
def test_individual_mode_tts_only(self, mock_esperanto, mock_env, client):
|
|
"""Test individual mode-specific var (TTS only)."""
|
|
|
|
# Mock environment: only TTS specific var is set
|
|
def env_side_effect(key):
|
|
if key == "OPENAI_COMPATIBLE_BASE_URL_TTS":
|
|
return "http://localhost:9000/v1"
|
|
return None
|
|
|
|
mock_env.side_effect = env_side_effect
|
|
|
|
# Mock Esperanto response
|
|
mock_esperanto.return_value = {
|
|
"language": ["openai-compatible"],
|
|
"embedding": ["openai-compatible"],
|
|
"speech_to_text": ["openai-compatible"],
|
|
"text_to_speech": ["openai-compatible"],
|
|
}
|
|
|
|
response = client.get("/api/models/providers")
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Should support only text_to_speech
|
|
supported = data["supported_types"]["openai_compatible"]
|
|
assert supported == ["text_to_speech"]
|
|
|
|
|
|
class TestUpdateDefaultModels:
|
|
"""PUT /models/defaults must distinguish 'field absent' (keep) from
|
|
'field explicitly null' (clear).
|
|
|
|
The handler used `is not None` guards, so a null sent to clear a default
|
|
was silently ignored — the old value survived while the client saw
|
|
success (#1091, same anti-pattern fixed for credentials in #1046). Now
|
|
keyed on model_fields_set, with the required defaults (chat, embedding)
|
|
rejecting explicit nulls.
|
|
"""
|
|
|
|
def _mock_defaults(self):
|
|
from unittest.mock import MagicMock
|
|
|
|
defaults = MagicMock()
|
|
defaults.default_chat_model = "model:chat"
|
|
defaults.default_transformation_model = "model:transform"
|
|
defaults.large_context_model = None
|
|
defaults.default_text_to_speech_model = "model:tts"
|
|
defaults.default_speech_to_text_model = None
|
|
defaults.default_embedding_model = "model:embed"
|
|
defaults.default_tools_model = "model:tools"
|
|
defaults.update = AsyncMock()
|
|
return defaults
|
|
|
|
def _put(self, client, defaults, body):
|
|
with patch(
|
|
"api.routers.models.DefaultModels.get_instance",
|
|
new=AsyncMock(return_value=defaults),
|
|
):
|
|
return client.put("/api/models/defaults", json=body)
|
|
|
|
def test_explicit_null_clears_optional_default(self, client):
|
|
defaults = self._mock_defaults()
|
|
response = self._put(client, defaults, {"default_transformation_model": None})
|
|
|
|
assert response.status_code == 200
|
|
assert defaults.default_transformation_model is None
|
|
defaults.update.assert_awaited_once()
|
|
assert response.json()["default_transformation_model"] is None
|
|
|
|
def test_absent_field_keeps_current_value(self, client):
|
|
defaults = self._mock_defaults()
|
|
response = self._put(client, defaults, {"default_tools_model": "model:new-tools"})
|
|
|
|
assert response.status_code == 200
|
|
assert defaults.default_tools_model == "model:new-tools"
|
|
# Not in the payload -> untouched (JSON null semantics must not leak in)
|
|
assert defaults.default_transformation_model == "model:transform"
|
|
assert response.json()["default_transformation_model"] == "model:transform"
|
|
|
|
def test_explicit_null_on_required_default_is_rejected(self, client):
|
|
for field in ("default_chat_model", "default_embedding_model"):
|
|
defaults = self._mock_defaults()
|
|
response = self._put(client, defaults, {field: None})
|
|
|
|
assert response.status_code == 400, field
|
|
assert field in response.json()["detail"]
|
|
defaults.update.assert_not_awaited()
|
|
|
|
def test_required_default_can_still_be_reassigned(self, client):
|
|
defaults = self._mock_defaults()
|
|
response = self._put(client, defaults, {"default_chat_model": "model:new-chat"})
|
|
|
|
assert response.status_code == 200
|
|
assert defaults.default_chat_model == "model:new-chat"
|
|
defaults.update.assert_awaited_once()
|