chore: import upstream snapshot with attribution
Test Migrations / Migrations (SQLite) (push) Has been cancelled
Build Dev Image / build-dev-image (push) Has been cancelled
Check i18n Keys / Check i18n Key Consistency (push) Has been cancelled
Lint / Ruff Lint & Format (push) Has been cancelled
Lint / Frontend Lint (push) Has been cancelled
Test Migrations / Migrations (PostgreSQL) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:44:22 +08:00
commit 534bb94eea
1288 changed files with 266913 additions and 0 deletions
View File
@@ -0,0 +1,203 @@
"""Tests for vector filter utilities."""
from __future__ import annotations
import pytest
from langbot.pkg.vector.filter_utils import (
SUPPORTED_OPS,
normalize_filter,
strip_unsupported_fields,
)
class TestNormalizeFilter:
"""Tests for normalize_filter function."""
def test_normalize_filter_empty_dict(self):
"""Empty dict returns empty list."""
result = normalize_filter({})
assert result == []
def test_normalize_filter_none(self):
"""None returns empty list."""
result = normalize_filter(None)
assert result == []
def test_normalize_filter_implicit_eq(self):
"""Bare value becomes implicit $eq."""
result = normalize_filter({'file_id': 'abc123'})
assert len(result) == 1
assert result[0] == ('file_id', '$eq', 'abc123')
def test_normalize_filter_explicit_eq(self):
"""Explicit $eq operator."""
result = normalize_filter({'file_id': {'$eq': 'abc123'}})
assert len(result) == 1
assert result[0] == ('file_id', '$eq', 'abc123')
def test_normalize_filter_comparison_operators(self):
"""Test comparison operators: $gt, $gte, $lt, $lte."""
result = normalize_filter({'created_at': {'$gte': 1700000000}})
assert len(result) == 1
assert result[0] == ('created_at', '$gte', 1700000000)
def test_normalize_filter_ne_operator(self):
"""Test $ne operator."""
result = normalize_filter({'status': {'$ne': 'deleted'}})
assert len(result) == 1
assert result[0] == ('status', '$ne', 'deleted')
def test_normalize_filter_in_operator(self):
"""Test $in operator with list value."""
result = normalize_filter({'file_type': {'$in': ['pdf', 'docx', 'txt']}})
assert len(result) == 1
assert result[0] == ('file_type', '$in', ['pdf', 'docx', 'txt'])
def test_normalize_filter_nin_operator(self):
"""Test $nin operator."""
result = normalize_filter({'status': {'$nin': ['deleted', 'archived']}})
assert len(result) == 1
assert result[0] == ('status', '$nin', ['deleted', 'archived'])
def test_normalize_filter_multiple_conditions(self):
"""Multiple top-level keys are AND-ed (returned as multiple triples)."""
result = normalize_filter({'file_id': 'abc', 'status': {'$ne': 'deleted'}, 'created_at': {'$gte': 1700000000}})
assert len(result) == 3
# Order should match dict iteration order
field_ops = [(field, op) for field, op, _ in result]
assert ('file_id', '$eq') in field_ops
assert ('status', '$ne') in field_ops
assert ('created_at', '$gte') in field_ops
def test_normalize_filter_unsupported_operator_raises(self):
"""Unsupported operator raises ValueError."""
with pytest.raises(ValueError, match='Unsupported filter operator'):
normalize_filter({'field': {'$regex': 'pattern'}})
def test_normalize_filter_all_supported_ops(self):
"""Test all supported operators are recognized."""
for op in SUPPORTED_OPS:
if op in ('$in', '$nin'):
filter_dict = {'field': {op: ['value1', 'value2']}}
else:
filter_dict = {'field': {op: 'value'}}
result = normalize_filter(filter_dict)
assert len(result) == 1
assert result[0][1] == op
class TestStripUnsupportedFields:
"""Tests for strip_unsupported_fields function."""
def test_strip_keeps_supported_fields(self):
"""Fields in supported_fields are kept."""
triples = [
('file_id', '$eq', 'abc'),
('chunk_uuid', '$ne', 'def'),
]
result = strip_unsupported_fields(triples, {'file_id', 'chunk_uuid'})
assert len(result) == 2
assert result == triples
def test_strip_removes_unsupported_fields(self):
"""Fields not in supported_fields are removed."""
triples = [
('file_id', '$eq', 'abc'),
('unknown_field', '$ne', 'def'),
]
result = strip_unsupported_fields(triples, {'file_id'})
assert len(result) == 1
assert result[0] == ('file_id', '$eq', 'abc')
def test_strip_empty_triples(self):
"""Empty triples list returns empty list."""
result = strip_unsupported_fields([], {'file_id'})
assert result == []
def test_strip_all_unsupported(self):
"""All fields unsupported returns empty list."""
triples = [
('unknown1', '$eq', 'a'),
('unknown2', '$eq', 'b'),
]
result = strip_unsupported_fields(triples, {'file_id'})
assert result == []
def test_strip_with_field_aliases(self):
"""Field aliases are resolved before checking support."""
triples = [
('uuid', '$eq', 'abc'), # alias for chunk_uuid
('file_id', '$eq', 'def'),
]
result = strip_unsupported_fields(triples, {'file_id', 'chunk_uuid'}, field_aliases={'uuid': 'chunk_uuid'})
assert len(result) == 2
# 'uuid' should be resolved to 'chunk_uuid'
assert result[0] == ('chunk_uuid', '$eq', 'abc')
assert result[1] == ('file_id', '$eq', 'def')
def test_strip_alias_not_in_supported(self):
"""Alias resolved but still not in supported_fields is dropped."""
triples = [
('uuid', '$eq', 'abc'), # alias for chunk_uuid, but not supported
]
result = strip_unsupported_fields(
triples,
{'file_id'}, # chunk_uuid not supported
field_aliases={'uuid': 'chunk_uuid'},
)
assert result == []
def test_strip_preserves_operator_and_value(self):
"""Strip only affects field name, not operator or value."""
triples = [
('file_id', '$in', ['a', 'b', 'c']),
]
result = strip_unsupported_fields(triples, {'file_id'})
assert result[0] == ('file_id', '$in', ['a', 'b', 'c'])
def test_strip_none_aliases(self):
"""None field_aliases is treated as empty dict."""
triples = [
('file_id', '$eq', 'abc'),
]
result = strip_unsupported_fields(triples, {'file_id'}, field_aliases=None)
assert len(result) == 1
assert result[0] == ('file_id', '$eq', 'abc')
class TestSupportedOpsConstant:
"""Tests for SUPPORTED_OPS constant."""
def test_supported_ops_contains_expected(self):
"""SUPPORTED_OPS contains all expected operators."""
expected = {'$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin'}
assert SUPPORTED_OPS == expected
def test_supported_ops_is_frozenset(self):
"""SUPPORTED_OPS is a frozenset for immutability."""
from collections.abc import Set
assert isinstance(SUPPORTED_OPS, Set)
+339
View File
@@ -0,0 +1,339 @@
"""Tests for VectorDBManager provider selection logic.
Tests the initialization logic that selects the appropriate VDB backend
based on configuration, without actually creating real VDB instances.
"""
from __future__ import annotations
from unittest.mock import MagicMock
from tests.utils.import_isolation import isolated_sys_modules
class TestVectorDBManagerInitialization:
"""Tests for VectorDBManager.initialize provider selection."""
def _create_mock_app(self, vdb_config: dict | None):
"""Create mock app with vdb configuration."""
mock_app = MagicMock()
mock_app.instance_config = MagicMock()
mock_app.instance_config.data = MagicMock()
mock_app.instance_config.data.get = MagicMock(return_value=vdb_config)
mock_app.logger = MagicMock()
mock_app.logger.info = MagicMock()
mock_app.logger.warning = MagicMock()
return mock_app
def _make_vector_import_mocks(self):
"""Create mocks for VDB backends to prevent real imports."""
mocks = {}
# Mock core.app to break circular import
mocks['langbot.pkg.core.app'] = MagicMock()
# Mock all VDB backend implementations
for backend in ['chroma', 'qdrant', 'seekdb', 'milvus', 'pgvector_db', 'valkey_search']:
mocks[f'langbot.pkg.vector.vdbs.{backend}'] = MagicMock()
return mocks
def test_initialize_no_config_defaults_to_chroma(self):
"""No vdb config defaults to Chroma."""
mock_app = self._create_mock_app(None)
mocks = self._make_vector_import_mocks()
# Create mock Chroma class
mock_chroma_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.chroma'].ChromaVectorDatabase = mock_chroma_class
with isolated_sys_modules(mocks):
# Import after mocking
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
# Run initialize synchronously for test
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
# Chroma should be instantiated
mock_chroma_class.assert_called_once_with(mock_app)
mock_app.logger.warning.assert_called()
def test_initialize_chroma_backend(self):
"""Explicit chroma config uses Chroma backend."""
vdb_config = {'use': 'chroma'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_chroma_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.chroma'].ChromaVectorDatabase = mock_chroma_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_chroma_class.assert_called_once_with(mock_app)
mock_app.logger.info.assert_called()
def test_initialize_qdrant_backend(self):
"""Qdrant config uses Qdrant backend."""
vdb_config = {'use': 'qdrant'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_qdrant_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.qdrant'].QdrantVectorDatabase = mock_qdrant_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_qdrant_class.assert_called_once_with(mock_app)
def test_initialize_seekdb_backend(self):
"""SeekDB config uses SeekDB backend."""
vdb_config = {'use': 'seekdb'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_seekdb_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.seekdb'].SeekDBVectorDatabase = mock_seekdb_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_seekdb_class.assert_called_once_with(mock_app)
def test_initialize_valkey_search_backend(self):
"""Valkey Search config uses ValkeySearchVectorDatabase backend."""
vdb_config = {'use': 'valkey_search'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_valkey_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.valkey_search'].ValkeySearchVectorDatabase = mock_valkey_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_valkey_class.assert_called_once_with(mock_app)
def test_initialize_milvus_backend_with_uri(self):
"""Milvus config with custom URI."""
vdb_config = {
'use': 'milvus',
'milvus': {'uri': 'http://localhost:19530', 'token': 'root:Milvus', 'db_name': 'langbot_db'},
}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_milvus_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.milvus'].MilvusVectorDatabase = mock_milvus_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_milvus_class.assert_called_once_with(
mock_app, uri='http://localhost:19530', token='root:Milvus', db_name='langbot_db'
)
def test_initialize_milvus_backend_defaults(self):
"""Milvus defaults when config not fully specified."""
vdb_config = {'use': 'milvus'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_milvus_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.milvus'].MilvusVectorDatabase = mock_milvus_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
# Should use default values
mock_milvus_class.assert_called_once_with(mock_app, uri='./data/milvus.db', token=None, db_name='default')
def test_initialize_pgvector_with_connection_string(self):
"""pgvector with connection string."""
vdb_config = {'use': 'pgvector', 'pgvector': {'connection_string': 'postgresql://user:pass@host:5432/langbot'}}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_pgvector_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.pgvector_db'].PgVectorDatabase = mock_pgvector_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_pgvector_class.assert_called_once_with(
mock_app, connection_string='postgresql://user:pass@host:5432/langbot'
)
def test_initialize_pgvector_with_individual_params(self):
"""pgvector with individual connection parameters."""
vdb_config = {
'use': 'pgvector',
'pgvector': {
'host': 'db.example.com',
'port': 5433,
'database': 'vectordb',
'user': 'admin',
'password': 'secret',
},
}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_pgvector_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.pgvector_db'].PgVectorDatabase = mock_pgvector_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_pgvector_class.assert_called_once_with(
mock_app, host='db.example.com', port=5433, database='vectordb', user='admin', password='secret'
)
def test_initialize_pgvector_defaults(self):
"""pgvector defaults when no config params."""
vdb_config = {'use': 'pgvector'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_pgvector_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.pgvector_db'].PgVectorDatabase = mock_pgvector_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_pgvector_class.assert_called_once_with(
mock_app, host='localhost', port=5432, database='langbot', user='postgres', password='postgres'
)
def test_initialize_unknown_backend_defaults_to_chroma(self):
"""Unknown vdb type defaults to Chroma with warning."""
vdb_config = {'use': 'unknown_backend'}
mock_app = self._create_mock_app(vdb_config)
mocks = self._make_vector_import_mocks()
mock_chroma_class = MagicMock()
mocks['langbot.pkg.vector.vdbs.chroma'].ChromaVectorDatabase = mock_chroma_class
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
import asyncio
asyncio.get_event_loop().run_until_complete(mgr.initialize())
mock_chroma_class.assert_called_once_with(mock_app)
mock_app.logger.warning.assert_called()
# Should warn about no valid backend
warning_msg = mock_app.logger.warning.call_args[0][0]
assert 'No valid' in warning_msg or 'defaulting' in warning_msg
class TestVectorDBManagerProxies:
"""Tests for VectorDBManager proxy methods."""
def test_get_supported_search_types_no_vector_db(self):
"""get_supported_search_types returns vector when no vector_db."""
mock_app = MagicMock()
mock_app.instance_config = MagicMock()
mock_app.instance_config.data = MagicMock()
mock_app.instance_config.data.get = MagicMock(return_value=None)
mock_app.logger = MagicMock()
mocks = {'langbot.pkg.core.app': MagicMock()}
for backend in ['chroma', 'qdrant', 'seekdb', 'milvus', 'pgvector_db']:
mocks[f'langbot.pkg.vector.vdbs.{backend}'] = MagicMock()
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
mgr.vector_db = None # Explicitly None
result = mgr.get_supported_search_types()
assert result == ['vector']
def test_get_supported_search_types_with_vector_db(self):
"""get_supported_search_types delegates to vector_db."""
mock_app = MagicMock()
# Create mock vector_db with supported_search_types
mock_vector_db = MagicMock()
mock_vector_db.supported_search_types = MagicMock(
return_value=[
MagicMock(value='vector'),
MagicMock(value='full_text'),
]
)
mocks = {'langbot.pkg.core.app': MagicMock()}
for backend in ['chroma', 'qdrant', 'seekdb', 'milvus', 'pgvector_db']:
mocks[f'langbot.pkg.vector.vdbs.{backend}'] = MagicMock()
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
mgr.vector_db = mock_vector_db
result = mgr.get_supported_search_types()
assert result == ['vector', 'full_text']
@@ -0,0 +1,391 @@
"""Unit tests for the Valkey Search VDB backend's pure helpers.
These tests exercise the filter-to-FT mapping, float32 packing, tag/text
escaping, FT.SEARCH reply parsing and the import guard. They run in the fast
CI lane and require NO running Valkey server.
"""
from __future__ import annotations
import asyncio
import struct
from importlib import import_module
from unittest.mock import AsyncMock
import pytest
def get_valkey_module():
"""Lazy import of the valkey_search backend module."""
return import_module('langbot.pkg.vector.vdbs.valkey_search')
def make_backend():
"""Construct a backend instance without running its __init__.
The constructor needs a live ``ap`` + config; for pure-helper tests we
only need a bare instance with the attributes the helpers touch.
"""
mod = get_valkey_module()
backend = object.__new__(mod.ValkeySearchVectorDatabase)
# _ensure_client serializes creation through this lock; set it here since
# __init__ (which normally creates it) is bypassed.
backend._client_lock = asyncio.Lock()
return backend
class TestFloat32Packing:
"""Tests for _pack_vector little-endian float32 packing."""
def test_pack_round_trips(self):
mod = get_valkey_module()
vec = [0.1, -2.5, 3.0, 4.25]
packed = mod.ValkeySearchVectorDatabase._pack_vector(vec)
assert isinstance(packed, bytes)
assert len(packed) == 4 * len(vec)
unpacked = list(struct.unpack(f'<{len(vec)}f', packed))
for original, restored in zip(vec, unpacked):
assert restored == pytest.approx(original, rel=1e-6)
def test_pack_is_little_endian(self):
mod = get_valkey_module()
packed = mod.ValkeySearchVectorDatabase._pack_vector([1.0])
assert packed == struct.pack('<f', 1.0)
class TestTagEscaping:
"""Tests for _escape_tag."""
def test_escapes_special_chars(self):
mod = get_valkey_module()
escaped = mod.ValkeySearchVectorDatabase._escape_tag('a-b c.d')
assert '\\-' in escaped
assert '\\ ' in escaped
assert '\\.' in escaped
def test_plain_value_unchanged(self):
mod = get_valkey_module()
assert mod.ValkeySearchVectorDatabase._escape_tag('abc123') == 'abc123'
class TestFileIdEncoding:
"""Tests for _encode_file_id (FT-unsafe char percent-encoding)."""
def test_uuid_is_noop(self):
mod = get_valkey_module()
fid = '550e8400-e29b-41d4-a716-446655440000'
assert mod.ValkeySearchVectorDatabase._encode_file_id(fid) == fid
def test_encodes_braces_star_and_percent(self):
mod = get_valkey_module()
enc = mod.ValkeySearchVectorDatabase._encode_file_id('a{b}c*d%e')
# '{'=7B '}'=7D '*'=2A '%'=25
assert enc == 'a%7Bb%7Dc%2Ad%25e'
# No raw FT-unsafe char survives.
assert all(ch not in enc for ch in '{}*') or '%' in enc
def test_encoding_is_deterministic_and_collision_safe(self):
mod = get_valkey_module()
enc = mod.ValkeySearchVectorDatabase._encode_file_id
# A literal "%7B" must not collide with an encoded "{".
assert enc('{') != enc('%7B')
assert enc('{') == '%7B'
assert enc('%7B') == '%257B'
def test_filter_encodes_unsafe_chars_in_tag_query(self):
backend = make_backend()
# The emitted TAG query must contain the encoded form, never raw braces.
frag = backend._triples_to_ft({'file_id': 'x}y{z*'})
assert '7D' in frag and '7B' in frag and '2A' in frag
# No raw '*' from the value, and exactly one opening/closing brace (the
# TAG-clause delimiters) — the value's own braces were encoded away.
assert '*' not in frag
assert frag.count('{') == 1 and frag.count('}') == 1
assert frag.startswith('@file_id:{') and frag.endswith('}')
def test_filter_in_operator_encodes_each_value(self):
backend = make_backend()
frag = backend._triples_to_ft({'file_id': {'$in': ['a*b', 'c}d']}})
assert '2A' in frag and '7D' in frag
assert '*' not in frag
class TestFilterToFt:
"""Tests for _triples_to_ft filter mapping (all 8 operators)."""
def test_empty_filter_returns_empty_string(self):
backend = make_backend()
assert backend._triples_to_ft(None) == ''
assert backend._triples_to_ft({}) == ''
def test_eq_tag(self):
backend = make_backend()
assert backend._triples_to_ft({'file_id': 'abc'}) == '@file_id:{abc}'
def test_explicit_eq_tag(self):
backend = make_backend()
assert backend._triples_to_ft({'file_id': {'$eq': 'abc'}}) == '@file_id:{abc}'
def test_ne_tag(self):
backend = make_backend()
assert backend._triples_to_ft({'file_id': {'$ne': 'abc'}}) == '-@file_id:{abc}'
def test_in_tag(self):
backend = make_backend()
assert backend._triples_to_ft({'file_id': {'$in': ['a', 'b']}}) == '@file_id:{a|b}'
def test_nin_tag(self):
backend = make_backend()
assert backend._triples_to_ft({'file_id': {'$nin': ['a', 'b']}}) == '-@file_id:{a|b}'
def test_numeric_range_operators(self):
backend = make_backend()
# file_id is the only indexed field; numeric ops still render via the
# generic range fragment, so use file_id to keep the field supported.
# Values are cast to float (defensive against non-numeric input and a
# future NUMERIC field becoming an injection surface).
assert backend._triples_to_ft({'file_id': {'$gt': 5}}) == '@file_id:[(5.0 +inf]'
assert backend._triples_to_ft({'file_id': {'$gte': 5}}) == '@file_id:[5.0 +inf]'
assert backend._triples_to_ft({'file_id': {'$lt': 5}}) == '@file_id:[-inf (5.0]'
assert backend._triples_to_ft({'file_id': {'$lte': 5}}) == '@file_id:[-inf 5.0]'
def test_numeric_range_rejects_non_numeric(self):
backend = make_backend()
# A non-numeric range value fails closed rather than interpolating raw.
with pytest.raises((ValueError, TypeError)):
backend._triples_to_ft({'file_id': {'$gt': 'not-a-number'}})
def test_unsupported_field_dropped(self):
backend = make_backend()
# Non-indexed fields are dropped (returns empty expression).
assert backend._triples_to_ft({'some_other_field': 'x'}) == ''
def test_multiple_supported_keys_anded(self):
backend = make_backend()
# Two conditions on the same indexed field are joined with a space (AND).
result = backend._triples_to_ft({'file_id': {'$in': ['a', 'b']}})
assert result == '@file_id:{a|b}'
class TestTextEscaping:
"""Tests for _escape_text full-text escaping."""
def test_escapes_ft_special_chars(self):
mod = get_valkey_module()
escaped = mod.ValkeySearchVectorDatabase._escape_text('hello@world|test')
assert '\\@' in escaped
assert '\\|' in escaped
class TestReplyToChroma:
"""Tests for _reply_to_chroma FT.SEARCH reply parsing."""
def test_parses_knn_reply(self):
backend = make_backend()
# glide returns [total, {key: {field: value}}]
reply = [
2,
{
b'kb:col1:id1': {
b'distance': b'0.10',
b'document': b'hello',
b'metadata_json': b'{"file_id": "f1"}',
},
b'kb:col1:id2': {
b'distance': b'0.25',
b'document': b'world',
b'metadata_json': b'{"file_id": "f2"}',
},
},
]
result = backend._reply_to_chroma('idx:col1', reply, has_distance=True)
assert result['ids'][0] == ['id1', 'id2']
assert result['distances'][0] == [pytest.approx(0.10), pytest.approx(0.25)]
assert result['metadatas'][0][0] == {'file_id': 'f1'}
assert result['metadatas'][0][1] == {'file_id': 'f2'}
def test_empty_reply(self):
backend = make_backend()
result = backend._reply_to_chroma('idx:col1', [0, {}], has_distance=True)
assert result == {'ids': [[]], 'metadatas': [[]], 'distances': [[]]}
def test_malformed_reply(self):
backend = make_backend()
result = backend._reply_to_chroma('idx:col1', [], has_distance=True)
assert result == {'ids': [[]], 'metadatas': [[]], 'distances': [[]]}
def test_text_search_reply_no_distance(self):
backend = make_backend()
reply = [
1,
{
b'kb:col1:id1': {
b'document': b'hello',
b'metadata_json': b'{"file_id": "f1"}',
},
},
]
result = backend._reply_to_chroma('idx:col1', reply, has_distance=False)
assert result['ids'][0] == ['id1']
assert result['distances'][0] == [0.0]
class TestImportGuard:
"""Tests for the ImportError guard when glide is unavailable."""
def test_constructor_raises_when_unavailable(self, monkeypatch):
mod = get_valkey_module()
monkeypatch.setattr(mod, 'VALKEY_SEARCH_AVAILABLE', False)
with pytest.raises(ImportError, match='valkey-glide'):
mod.ValkeySearchVectorDatabase(ap=None)
class TestSupportedSearchTypes:
"""Tests for supported_search_types."""
def test_supports_vector_full_text_hybrid(self):
mod = get_valkey_module()
from langbot.pkg.vector.vdb import SearchType
types = mod.ValkeySearchVectorDatabase.supported_search_types()
assert SearchType.VECTOR in types
assert SearchType.FULL_TEXT in types
assert SearchType.HYBRID in types
class TestDeleteByFilterGuard:
"""Regression tests for the delete_by_filter mass-deletion guard.
A non-empty filter referencing only non-indexed fields must NOT fall back
to match-all and wipe the whole collection: it must skip and return 0.
"""
async def test_unsupported_only_filter_skips_and_returns_zero(self):
backend = make_backend()
# Make the client/index lookups succeed without a real server.
backend._client = AsyncMock()
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
backend._ensure_client = AsyncMock(return_value=backend._client)
backend._index_exists = AsyncMock(return_value=True)
# _search_keys must never be reached for an unusable filter.
backend._search_keys = AsyncMock(
side_effect=AssertionError('_search_keys must not be called for an unusable filter')
)
# Filter references only a non-indexed field -> maps to no FT conditions.
deleted = await backend.delete_by_filter('col1', {'some_other_field': 'x'})
assert deleted == 0
backend._client.delete.assert_not_called()
async def test_supported_filter_deletes_matching_keys(self):
backend = make_backend()
backend._client = AsyncMock()
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
backend._ensure_client = AsyncMock(return_value=backend._client)
backend._index_exists = AsyncMock(return_value=True)
backend._search_keys = AsyncMock(return_value=['kb:col1:id1', 'kb:col1:id2'])
deleted = await backend.delete_by_filter('col1', {'file_id': 'f1'})
assert deleted == 2
backend._client.delete.assert_awaited_once_with(['kb:col1:id1', 'kb:col1:id2'])
class TestClose:
"""Tests for the close() teardown."""
async def test_close_resets_client_and_indexes(self):
backend = make_backend()
client = AsyncMock()
backend._client = client
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
backend._ensured_indexes = {'idx:col1'}
await backend.close()
client.close.assert_awaited_once()
assert backend._client is None
assert backend._ensured_indexes == set()
async def test_close_is_noop_when_no_client(self):
backend = make_backend()
backend._client = None
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
backend._ensured_indexes = set()
# Should not raise.
await backend.close()
assert backend._client is None
class TestCredentialsBuild:
"""Tests for the auth-credential construction in _ensure_client."""
def _prep_backend(self, mod, monkeypatch, *, username, password):
backend = make_backend()
backend._client = None
backend._host = 'localhost'
backend._port = 6379
backend._db = 0
backend._tls = False
backend._username = username
backend._password = password
backend._request_timeout = 5000
backend._ensured_indexes = set()
warnings: list[str] = []
backend.ap = type(
'Ap',
(),
{
'logger': type(
'L', (), {'info': lambda self, *a, **k: None, 'warning': lambda s, m, *a, **k: warnings.append(m)}
)()
},
)()
created = {}
class _FakeClient:
@staticmethod
async def create(conf):
created['conf'] = conf
return AsyncMock()
cred_calls: list[dict] = []
def _fake_credentials(**kwargs):
cred_calls.append(kwargs)
return ('CRED', kwargs)
# These names are absent when the optional valkey-glide dependency is
# unavailable (for example, on Windows), so allow the test doubles to
# create them on the module.
monkeypatch.setattr(mod, 'GlideClient', _FakeClient, raising=False)
monkeypatch.setattr(mod, 'ServerCredentials', _fake_credentials, raising=False)
monkeypatch.setattr(mod, 'GlideClientConfiguration', lambda **kw: kw, raising=False)
monkeypatch.setattr(mod, 'NodeAddress', lambda *a, **k: ('node', a, k), raising=False)
return backend, created, cred_calls, warnings
async def test_username_without_password_fails_closed(self, monkeypatch):
mod = get_valkey_module()
backend, created, cred_calls, warnings = self._prep_backend(mod, monkeypatch, username='acluser', password=None)
# A username without a password must fail closed rather than silently
# connecting unauthenticated to a (potentially shared) Valkey instance.
with pytest.raises(ValueError, match='without a password'):
await backend._ensure_client()
assert cred_calls == [] # ServerCredentials NOT constructed
assert 'conf' not in created # client never created
async def test_password_builds_credentials(self, monkeypatch):
mod = get_valkey_module()
backend, created, cred_calls, warnings = self._prep_backend(
mod, monkeypatch, username='acluser', password='secret'
)
await backend._ensure_client()
assert len(cred_calls) == 1
assert cred_calls[0] == {'password': 'secret', 'username': 'acluser'}
assert created['conf']['credentials'] == ('CRED', {'password': 'secret', 'username': 'acluser'})
+205
View File
@@ -0,0 +1,205 @@
"""Tests for VectorDatabase base class and SearchType enum."""
from __future__ import annotations
from unittest.mock import AsyncMock
import pytest
from langbot.pkg.vector.vdb import SearchType, VectorDatabase
class TestSearchType:
"""Tests for SearchType enum."""
def test_search_type_values(self):
"""Test SearchType enum values."""
assert SearchType.VECTOR.value == 'vector'
assert SearchType.FULL_TEXT.value == 'full_text'
assert SearchType.HYBRID.value == 'hybrid'
def test_search_type_is_string_enum(self):
"""SearchType is a string enum."""
assert isinstance(SearchType.VECTOR, str)
assert SearchType.VECTOR == 'vector'
def test_search_type_from_string(self):
"""Can create SearchType from string."""
assert SearchType('vector') == SearchType.VECTOR
assert SearchType('full_text') == SearchType.FULL_TEXT
assert SearchType('hybrid') == SearchType.HYBRID
class TestVectorDatabaseAbstractMethods:
"""Tests for VectorDatabase abstract methods."""
def test_vector_database_is_abstract(self):
"""VectorDatabase is abstract and cannot be instantiated directly."""
with pytest.raises(TypeError):
VectorDatabase()
def test_abstract_methods_required(self):
"""Subclass must implement all abstract methods."""
class IncompleteVectorDB(VectorDatabase):
pass
with pytest.raises(TypeError):
IncompleteVectorDB()
def test_supported_search_types_default(self):
"""Default supported_search_types returns [VECTOR]."""
class MinimalVectorDB(VectorDatabase):
async def add_embeddings(self, collection, ids, embeddings_list, metadatas, documents=None):
pass
async def search(
self,
collection,
query_embedding,
k=5,
search_type='vector',
query_text='',
filter=None,
vector_weight=None,
):
pass
async def delete_by_file_id(self, collection, file_id):
pass
async def delete_by_filter(self, collection, filter):
pass
async def get_or_create_collection(self, collection):
pass
async def delete_collection(self, collection):
pass
db = MinimalVectorDB()
assert db.supported_search_types() == [SearchType.VECTOR]
def test_list_by_filter_default_implementation(self):
"""list_by_filter has default implementation returning empty."""
class MinimalVectorDB(VectorDatabase):
async def add_embeddings(self, collection, ids, embeddings_list, metadatas, documents=None):
pass
async def search(
self,
collection,
query_embedding,
k=5,
search_type='vector',
query_text='',
filter=None,
vector_weight=None,
):
pass
async def delete_by_file_id(self, collection, file_id):
pass
async def delete_by_filter(self, collection, filter):
pass
async def get_or_create_collection(self, collection):
pass
async def delete_collection(self, collection):
pass
db = MinimalVectorDB()
# list_by_filter should return empty list and -1 for total
import asyncio
result = asyncio.get_event_loop().run_until_complete(db.list_by_filter('test_collection'))
assert result == ([], -1)
class TestVectorDatabaseInterface:
"""Tests for VectorDatabase interface contracts."""
@pytest.fixture
def mock_vector_db(self):
"""Create a minimal mock VectorDatabase for testing."""
class MockVectorDB(VectorDatabase):
def __init__(self):
self.add_embeddings = AsyncMock()
self.search = AsyncMock(
return_value={
'ids': [['id1', 'id2']],
'distances': [[0.1, 0.2]],
'metadatas': [[{'key': 'val1'}, {'key': 'val2'}]],
}
)
self.delete_by_file_id = AsyncMock()
self.delete_by_filter = AsyncMock(return_value=5)
self.get_or_create_collection = AsyncMock()
self.delete_collection = AsyncMock()
async def add_embeddings(self, collection, ids, embeddings_list, metadatas, documents=None):
pass
async def search(
self,
collection,
query_embedding,
k=5,
search_type='vector',
query_text='',
filter=None,
vector_weight=None,
):
pass
async def delete_by_file_id(self, collection, file_id):
pass
async def delete_by_filter(self, collection, filter):
pass
async def get_or_create_collection(self, collection):
pass
async def delete_collection(self, collection):
pass
return MockVectorDB()
@pytest.mark.asyncio
async def test_add_embeddings_signature(self, mock_vector_db):
"""add_embeddings has expected signature."""
await mock_vector_db.add_embeddings(
collection='test',
ids=['id1', 'id2'],
embeddings_list=[[0.1, 0.2], [0.3, 0.4]],
metadatas=[{'a': 1}, {'b': 2}],
documents=['doc1', 'doc2'],
)
mock_vector_db.add_embeddings.assert_called_once()
@pytest.mark.asyncio
async def test_search_signature(self, mock_vector_db):
"""search has expected signature with all optional params."""
import numpy as np
await mock_vector_db.search(
collection='test',
query_embedding=np.array([0.1, 0.2]),
k=10,
search_type='hybrid',
query_text='search text',
filter={'file_id': 'abc'},
vector_weight=0.7,
)
mock_vector_db.search.assert_called_once()
@pytest.mark.asyncio
async def test_delete_by_filter_returns_int(self, mock_vector_db):
"""delete_by_filter returns int count."""
result = await mock_vector_db.delete_by_filter('test', {'file_id': 'abc'})
assert isinstance(result, int)
@@ -0,0 +1,369 @@
"""Tests for VDB backend filter conversion functions.
Tests cover:
- _build_qdrant_filter: Qdrant models.Filter conversion
- _build_milvus_expr: Milvus boolean expression string conversion
- _build_pg_conditions: PostgreSQL SQLAlchemy conditions conversion
"""
from __future__ import annotations
from importlib import import_module
def get_qdrant_module():
"""Lazy import qdrant module."""
return import_module('langbot.pkg.vector.vdbs.qdrant')
def get_milvus_module():
"""Lazy import milvus module."""
return import_module('langbot.pkg.vector.vdbs.milvus')
def get_pgvector_module():
"""Lazy import pgvector module."""
return import_module('langbot.pkg.vector.vdbs.pgvector_db')
class TestQdrantFilterConversion:
"""Tests for _build_qdrant_filter function."""
def test_empty_filter_returns_empty_must(self):
"""Empty filter dict returns Filter with None must/must_not."""
qdrant_module = get_qdrant_module()
result = qdrant_module._build_qdrant_filter({})
assert result.must is None
assert result.must_not is None
def test_eq_operator_creates_must_condition(self):
"""$eq operator creates FieldCondition in must list."""
qdrant_module = get_qdrant_module()
from qdrant_client import models
result = qdrant_module._build_qdrant_filter({'file_id': 'abc'})
assert result.must is not None
assert len(result.must) == 1
condition = result.must[0]
assert condition.key == 'file_id'
assert isinstance(condition.match, models.MatchValue)
assert condition.match.value == 'abc'
def test_ne_operator_creates_must_not_condition(self):
"""$ne operator creates FieldCondition in must_not list."""
qdrant_module = get_qdrant_module()
from qdrant_client import models
result = qdrant_module._build_qdrant_filter({'status': {'$ne': 'deleted'}})
assert result.must_not is not None
assert len(result.must_not) == 1
condition = result.must_not[0]
assert condition.key == 'status'
assert isinstance(condition.match, models.MatchValue)
assert condition.match.value == 'deleted'
def test_in_operator_creates_match_any(self):
"""$in operator creates MatchAny condition."""
qdrant_module = get_qdrant_module()
from qdrant_client import models
result = qdrant_module._build_qdrant_filter({'file_type': {'$in': ['pdf', 'docx']}})
assert result.must is not None
assert len(result.must) == 1
condition = result.must[0]
assert condition.key == 'file_type'
assert isinstance(condition.match, models.MatchAny)
assert condition.match.any == ['pdf', 'docx']
def test_nin_operator_creates_must_not_match_any(self):
"""$nin operator creates MatchAny in must_not."""
qdrant_module = get_qdrant_module()
from qdrant_client import models
result = qdrant_module._build_qdrant_filter({'status': {'$nin': ['deleted', 'archived']}})
assert result.must_not is not None
assert len(result.must_not) == 1
condition = result.must_not[0]
assert condition.key == 'status'
assert isinstance(condition.match, models.MatchAny)
assert condition.match.any == ['deleted', 'archived']
def test_range_operators_create_range_condition(self):
"""$gt, $gte, $lt, $lte create Range conditions."""
qdrant_module = get_qdrant_module()
from qdrant_client import models
# Test $gt
result = qdrant_module._build_qdrant_filter({'created_at': {'$gt': 100}})
condition = result.must[0]
assert isinstance(condition.range, models.Range)
assert condition.range.gt == 100
# Test $gte
result = qdrant_module._build_qdrant_filter({'created_at': {'$gte': 100}})
condition = result.must[0]
assert condition.range.gte == 100
# Test $lt
result = qdrant_module._build_qdrant_filter({'created_at': {'$lt': 100}})
condition = result.must[0]
assert condition.range.lt == 100
# Test $lte
result = qdrant_module._build_qdrant_filter({'created_at': {'$lte': 100}})
condition = result.must[0]
assert condition.range.lte == 100
def test_multiple_conditions_combined(self):
"""Multiple conditions are combined in must/must_not."""
qdrant_module = get_qdrant_module()
result = qdrant_module._build_qdrant_filter(
{
'file_id': 'abc',
'status': {'$ne': 'deleted'},
'created_at': {'$gte': 100},
}
)
assert len(result.must) == 2 # file_id eq + created_at gte
assert len(result.must_not) == 1 # status ne
def test_implicit_eq_handled(self):
"""Implicit $eq (bare value) is correctly handled."""
qdrant_module = get_qdrant_module()
from qdrant_client import models
result = qdrant_module._build_qdrant_filter({'field': 'value'})
assert result.must is not None
condition = result.must[0]
assert isinstance(condition.match, models.MatchValue)
class TestMilvusFilterConversion:
"""Tests for _build_milvus_expr function.
NOTE: Milvus only supports fields: 'text', 'file_id', 'chunk_uuid'
Tests use only these supported fields.
"""
def test_empty_filter_returns_empty_string(self):
"""Empty filter dict returns empty string."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({})
assert result == ''
def test_eq_operator_expression(self):
"""$eq operator creates == expression."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'file_id': 'abc'})
assert result == 'file_id == "abc"'
def test_ne_operator_expression(self):
"""$ne operator creates != expression."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'file_id': {'$ne': 'deleted'}})
assert result == 'file_id != "deleted"'
def test_comparison_operators(self):
"""$gt, $gte, $lt, $lte create comparison expressions."""
milvus_module = get_milvus_module()
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$gt': 'uuid_100'}}) == 'chunk_uuid > "uuid_100"'
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$gte': 'uuid_100'}}) == 'chunk_uuid >= "uuid_100"'
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$lt': 'uuid_100'}}) == 'chunk_uuid < "uuid_100"'
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$lte': 'uuid_100'}}) == 'chunk_uuid <= "uuid_100"'
def test_in_operator_expression(self):
"""$in operator creates in [...] expression."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'file_id': {'$in': ['pdf', 'docx']}})
assert result == 'file_id in ["pdf", "docx"]'
def test_nin_operator_expression(self):
"""$nin operator creates not in [...] expression."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'file_id': {'$nin': ['deleted', 'archived']}})
assert result == 'file_id not in ["deleted", "archived"]'
def test_multiple_conditions_joined_with_and(self):
"""Multiple conditions are joined with 'and'."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr(
{
'file_id': 'abc',
'chunk_uuid': {'$ne': 'def'},
}
)
assert 'and' in result
assert 'file_id == "abc"' in result
assert 'chunk_uuid != "def"' in result
def test_string_value_escaped(self):
"""String values are properly escaped."""
milvus_module = get_milvus_module()
# Test backslash escape
result = milvus_module._build_milvus_expr({'file_id': 'C:\\Users\\test'})
assert '\\\\' in result
# Test quote escape
result = milvus_module._build_milvus_expr({'file_id': 'test "quoted"'})
assert '\\"' in result
def test_text_field_supported(self):
"""text field is supported."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'text': 'some text'})
assert result == 'text == "some text"'
def test_milvus_literal_function(self):
"""Test _milvus_literal helper."""
milvus_module = get_milvus_module()
assert milvus_module._milvus_literal('string') == '"string"'
assert milvus_module._milvus_literal(42) == '42'
assert milvus_module._milvus_literal(3.14) == '3.14'
def test_unsupported_field_dropped(self):
"""Unsupported fields are dropped (not in _MILVUS_SUPPORTED_FIELDS)."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'unknown_field': 'value'})
assert result == ''
def test_uuid_alias_resolved(self):
"""'uuid' alias is resolved to 'chunk_uuid'."""
milvus_module = get_milvus_module()
result = milvus_module._build_milvus_expr({'uuid': 'abc'})
assert result.startswith('chunk_uuid')
# uuid substring appears in chunk_uuid which is expected
class TestPgVectorFilterConversion:
"""Tests for _build_pg_conditions function.
NOTE: PGVector only supports fields: 'text', 'file_id', 'chunk_uuid'
Tests use only these supported fields.
"""
def test_empty_filter_returns_empty_list(self):
"""Empty filter dict returns empty list."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({})
assert result == []
def test_eq_operator_creates_equality_condition(self):
"""$eq operator creates SQLAlchemy == condition."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({'file_id': 'abc'})
assert len(result) == 1
# Verify it's a SQLAlchemy BinaryExpression
from sqlalchemy.sql.expression import BinaryExpression
assert isinstance(result[0], BinaryExpression)
def test_ne_operator_creates_inequality_condition(self):
"""$ne operator creates SQLAlchemy != condition."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({'file_id': {'$ne': 'deleted'}})
assert len(result) == 1
# Operator should be ne (not equals)
assert '!=' in str(result[0]) or 'ne' in str(result[0].operator)
def test_comparison_operators(self):
"""$gt, $gte, $lt, $lte create comparison conditions."""
pgvector_module = get_pgvector_module()
# Test all comparison operators with supported field
for op, expected_op in [
('$gt', '>'),
('$gte', '>='),
('$lt', '<'),
('$lte', '<='),
]:
result = pgvector_module._build_pg_conditions({'chunk_uuid': {op: 'uuid_100'}})
assert len(result) == 1
assert expected_op in str(result[0])
def test_in_operator_creates_in_condition(self):
"""$in operator creates SQLAlchemy in_ condition."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({'file_id': {'$in': ['a', 'b', 'c']}})
assert len(result) == 1
assert 'IN' in str(result[0]).upper()
def test_nin_operator_creates_notin_condition(self):
"""$nin operator creates SQLAlchemy notin_ condition."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({'file_id': {'$nin': ['a', 'b']}})
assert len(result) == 1
assert 'NOT IN' in str(result[0]).upper()
def test_multiple_conditions_list(self):
"""Multiple conditions return list of conditions."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions(
{
'file_id': 'abc',
'chunk_uuid': {'$ne': 'def'},
}
)
assert len(result) == 2
def test_unsupported_field_dropped(self):
"""Unsupported fields are dropped (not in _PG_SUPPORTED_FIELDS)."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({'unknown_field': 'value'})
assert result == []
def test_uuid_alias_resolved(self):
"""'uuid' alias is resolved to 'chunk_uuid'."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions({'uuid': 'abc'})
assert len(result) == 1
# Should reference chunk_uuid column
assert 'chunk_uuid' in str(result[0])
def test_supported_fields_only(self):
"""Only supported fields (text, file_id, chunk_uuid) are kept."""
pgvector_module = get_pgvector_module()
result = pgvector_module._build_pg_conditions(
{
'text': {'$ne': ''},
'file_id': 'abc',
'chunk_uuid': {'$in': ['x', 'y']},
'unsupported': 'value',
}
)
assert len(result) == 3 # Only supported fields