wehub-resource-sync
138 lines
5.1 KiB
Python
138 lines
5.1 KiB
Python
"""
|
|
Tests for connection-test error classification (open_notebook/ai/connection_tester.py).
|
|
|
|
Two semantics share one auth/network classifier:
|
|
- The *provider* test asks only "do these credentials reach a working
|
|
provider?" — so a missing/retired/unsupported test model is SUCCESS (the
|
|
#970 durability fix): a hard-coded test model going away must never be
|
|
misreported as a broken connection.
|
|
- The *individual model* test validates one specific registered model, so a
|
|
missing model there IS a failure.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from open_notebook.ai.connection_tester import (
|
|
_connection_failure_reason,
|
|
_is_rate_limit,
|
|
_normalize_error_message,
|
|
classify_provider_test_error,
|
|
)
|
|
|
|
# Realistic provider error strings.
|
|
GOOGLE_RETIRED_MODEL_404 = (
|
|
"404 models/gemini-2.0-flash is not found for API version v1beta, "
|
|
"or is not supported for generateContent."
|
|
)
|
|
GOOGLE_DEPRECATED = "400 Model gemini-1.5-pro has been deprecated."
|
|
GOOGLE_BAD_KEY_401 = "401 API key not valid. Please pass a valid API key."
|
|
GOOGLE_PERM_403 = "403 Permission denied on resource project."
|
|
GOOGLE_QUOTA_429 = "429 Resource has been exhausted (e.g. check quota)."
|
|
DNS_FAILURE = "Connection error: [Errno -2] Name or service not known (getaddrinfo failed)"
|
|
TIMEOUT = "Request timed out after 10s"
|
|
|
|
|
|
class TestConnectionFailureReason:
|
|
"""Only auth/permission/network are true connection failures."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"msg,expected_fragment",
|
|
[
|
|
(GOOGLE_BAD_KEY_401, "Invalid API key"),
|
|
(GOOGLE_PERM_403, "lacks required permissions"),
|
|
(DNS_FAILURE, "Connection error"),
|
|
(TIMEOUT, "timed out"),
|
|
],
|
|
)
|
|
def test_real_failures_return_a_reason(self, msg, expected_fragment):
|
|
reason = _connection_failure_reason(msg)
|
|
assert reason is not None
|
|
assert expected_fragment in reason
|
|
|
|
@pytest.mark.parametrize(
|
|
"msg",
|
|
[GOOGLE_RETIRED_MODEL_404, GOOGLE_DEPRECATED, GOOGLE_QUOTA_429],
|
|
)
|
|
def test_provider_reached_returns_none(self, msg):
|
|
# A model/quota problem came back FROM the provider — not a failure.
|
|
assert _connection_failure_reason(msg) is None
|
|
|
|
|
|
class TestRateLimitDetection:
|
|
@pytest.mark.parametrize(
|
|
"msg",
|
|
[
|
|
"429 Too Many Requests",
|
|
"Resource has been exhausted (e.g. check quota).",
|
|
"You have hit your rate limit",
|
|
"quota exceeded for this project",
|
|
],
|
|
)
|
|
def test_detects_throttling_phrasings(self, msg):
|
|
assert _is_rate_limit(msg) is True
|
|
|
|
def test_plain_model_error_is_not_rate_limit(self):
|
|
assert _is_rate_limit(GOOGLE_RETIRED_MODEL_404) is False
|
|
|
|
|
|
class TestClassifyProviderTestError:
|
|
"""Provider test: only bad creds / unreachable endpoint fail."""
|
|
|
|
def test_retired_test_model_is_success(self):
|
|
# The core #970 regression: a shut-down test model must report the
|
|
# key as valid, not the connection as broken.
|
|
success, message = classify_provider_test_error(GOOGLE_RETIRED_MODEL_404)
|
|
assert success is True
|
|
assert "test model unavailable" in message
|
|
|
|
def test_deprecated_model_is_success(self):
|
|
success, _ = classify_provider_test_error(GOOGLE_DEPRECATED)
|
|
assert success is True
|
|
|
|
def test_rate_limit_is_success(self):
|
|
success, message = classify_provider_test_error(GOOGLE_QUOTA_429)
|
|
assert success is True
|
|
assert "connection works" in message
|
|
|
|
@pytest.mark.parametrize(
|
|
"msg,expected_fragment",
|
|
[
|
|
(GOOGLE_BAD_KEY_401, "Invalid API key"),
|
|
(GOOGLE_PERM_403, "lacks required permissions"),
|
|
(DNS_FAILURE, "Connection error"),
|
|
(TIMEOUT, "timed out"),
|
|
],
|
|
)
|
|
def test_auth_and_network_still_fail(self, msg, expected_fragment):
|
|
success, message = classify_provider_test_error(msg)
|
|
assert success is False
|
|
assert expected_fragment in message
|
|
|
|
def test_unrecognized_error_stays_a_failure(self):
|
|
# A construction/config error we can't attribute to the provider is
|
|
# surfaced, not silently reported as success.
|
|
success, message = classify_provider_test_error(
|
|
"TypeError: create_language() missing required config 'base_url'"
|
|
)
|
|
assert success is False
|
|
assert "Error:" in message
|
|
|
|
|
|
class TestIndividualModelSemanticsDiffer:
|
|
"""The SAME model-not-found string is failure for the individual-model
|
|
test but success for the provider test — the intended difference."""
|
|
|
|
def test_model_not_found_is_failure_for_individual_test(self):
|
|
success, message = _normalize_error_message(GOOGLE_RETIRED_MODEL_404)
|
|
assert success is False
|
|
assert "Model not found" in message
|
|
|
|
def test_same_string_is_success_for_provider_test(self):
|
|
success, _ = classify_provider_test_error(GOOGLE_RETIRED_MODEL_404)
|
|
assert success is True
|
|
|
|
def test_individual_test_shares_auth_classification(self):
|
|
success, message = _normalize_error_message(GOOGLE_BAD_KEY_401)
|
|
assert success is False
|
|
assert "Invalid API key" in message
|