chore: import upstream snapshot with attribution
Test Migrations / Migrations (SQLite) (push) Has been cancelled
Build Dev Image / build-dev-image (push) Has been cancelled
Check i18n Keys / Check i18n Key Consistency (push) Has been cancelled
Lint / Ruff Lint & Format (push) Has been cancelled
Lint / Frontend Lint (push) Has been cancelled
Test Migrations / Migrations (PostgreSQL) (push) Has been cancelled
Test Migrations / Migrations (SQLite) (push) Has been cancelled
Build Dev Image / build-dev-image (push) Has been cancelled
Check i18n Keys / Check i18n Key Consistency (push) Has been cancelled
Lint / Ruff Lint & Format (push) Has been cancelled
Lint / Frontend Lint (push) Has been cancelled
Test Migrations / Migrations (PostgreSQL) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
"""Tests for vector filter utilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from langbot.pkg.vector.filter_utils import (
|
||||
SUPPORTED_OPS,
|
||||
normalize_filter,
|
||||
strip_unsupported_fields,
|
||||
)
|
||||
|
||||
|
||||
class TestNormalizeFilter:
|
||||
"""Tests for normalize_filter function."""
|
||||
|
||||
def test_normalize_filter_empty_dict(self):
|
||||
"""Empty dict returns empty list."""
|
||||
result = normalize_filter({})
|
||||
assert result == []
|
||||
|
||||
def test_normalize_filter_none(self):
|
||||
"""None returns empty list."""
|
||||
result = normalize_filter(None)
|
||||
assert result == []
|
||||
|
||||
def test_normalize_filter_implicit_eq(self):
|
||||
"""Bare value becomes implicit $eq."""
|
||||
result = normalize_filter({'file_id': 'abc123'})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('file_id', '$eq', 'abc123')
|
||||
|
||||
def test_normalize_filter_explicit_eq(self):
|
||||
"""Explicit $eq operator."""
|
||||
result = normalize_filter({'file_id': {'$eq': 'abc123'}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('file_id', '$eq', 'abc123')
|
||||
|
||||
def test_normalize_filter_comparison_operators(self):
|
||||
"""Test comparison operators: $gt, $gte, $lt, $lte."""
|
||||
result = normalize_filter({'created_at': {'$gte': 1700000000}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('created_at', '$gte', 1700000000)
|
||||
|
||||
def test_normalize_filter_ne_operator(self):
|
||||
"""Test $ne operator."""
|
||||
result = normalize_filter({'status': {'$ne': 'deleted'}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('status', '$ne', 'deleted')
|
||||
|
||||
def test_normalize_filter_in_operator(self):
|
||||
"""Test $in operator with list value."""
|
||||
result = normalize_filter({'file_type': {'$in': ['pdf', 'docx', 'txt']}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('file_type', '$in', ['pdf', 'docx', 'txt'])
|
||||
|
||||
def test_normalize_filter_nin_operator(self):
|
||||
"""Test $nin operator."""
|
||||
result = normalize_filter({'status': {'$nin': ['deleted', 'archived']}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('status', '$nin', ['deleted', 'archived'])
|
||||
|
||||
def test_normalize_filter_multiple_conditions(self):
|
||||
"""Multiple top-level keys are AND-ed (returned as multiple triples)."""
|
||||
result = normalize_filter({'file_id': 'abc', 'status': {'$ne': 'deleted'}, 'created_at': {'$gte': 1700000000}})
|
||||
|
||||
assert len(result) == 3
|
||||
# Order should match dict iteration order
|
||||
field_ops = [(field, op) for field, op, _ in result]
|
||||
assert ('file_id', '$eq') in field_ops
|
||||
assert ('status', '$ne') in field_ops
|
||||
assert ('created_at', '$gte') in field_ops
|
||||
|
||||
def test_normalize_filter_unsupported_operator_raises(self):
|
||||
"""Unsupported operator raises ValueError."""
|
||||
with pytest.raises(ValueError, match='Unsupported filter operator'):
|
||||
normalize_filter({'field': {'$regex': 'pattern'}})
|
||||
|
||||
def test_normalize_filter_all_supported_ops(self):
|
||||
"""Test all supported operators are recognized."""
|
||||
for op in SUPPORTED_OPS:
|
||||
if op in ('$in', '$nin'):
|
||||
filter_dict = {'field': {op: ['value1', 'value2']}}
|
||||
else:
|
||||
filter_dict = {'field': {op: 'value'}}
|
||||
|
||||
result = normalize_filter(filter_dict)
|
||||
assert len(result) == 1
|
||||
assert result[0][1] == op
|
||||
|
||||
|
||||
class TestStripUnsupportedFields:
|
||||
"""Tests for strip_unsupported_fields function."""
|
||||
|
||||
def test_strip_keeps_supported_fields(self):
|
||||
"""Fields in supported_fields are kept."""
|
||||
triples = [
|
||||
('file_id', '$eq', 'abc'),
|
||||
('chunk_uuid', '$ne', 'def'),
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(triples, {'file_id', 'chunk_uuid'})
|
||||
|
||||
assert len(result) == 2
|
||||
assert result == triples
|
||||
|
||||
def test_strip_removes_unsupported_fields(self):
|
||||
"""Fields not in supported_fields are removed."""
|
||||
triples = [
|
||||
('file_id', '$eq', 'abc'),
|
||||
('unknown_field', '$ne', 'def'),
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(triples, {'file_id'})
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('file_id', '$eq', 'abc')
|
||||
|
||||
def test_strip_empty_triples(self):
|
||||
"""Empty triples list returns empty list."""
|
||||
result = strip_unsupported_fields([], {'file_id'})
|
||||
assert result == []
|
||||
|
||||
def test_strip_all_unsupported(self):
|
||||
"""All fields unsupported returns empty list."""
|
||||
triples = [
|
||||
('unknown1', '$eq', 'a'),
|
||||
('unknown2', '$eq', 'b'),
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(triples, {'file_id'})
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_strip_with_field_aliases(self):
|
||||
"""Field aliases are resolved before checking support."""
|
||||
triples = [
|
||||
('uuid', '$eq', 'abc'), # alias for chunk_uuid
|
||||
('file_id', '$eq', 'def'),
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(triples, {'file_id', 'chunk_uuid'}, field_aliases={'uuid': 'chunk_uuid'})
|
||||
|
||||
assert len(result) == 2
|
||||
# 'uuid' should be resolved to 'chunk_uuid'
|
||||
assert result[0] == ('chunk_uuid', '$eq', 'abc')
|
||||
assert result[1] == ('file_id', '$eq', 'def')
|
||||
|
||||
def test_strip_alias_not_in_supported(self):
|
||||
"""Alias resolved but still not in supported_fields is dropped."""
|
||||
triples = [
|
||||
('uuid', '$eq', 'abc'), # alias for chunk_uuid, but not supported
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(
|
||||
triples,
|
||||
{'file_id'}, # chunk_uuid not supported
|
||||
field_aliases={'uuid': 'chunk_uuid'},
|
||||
)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_strip_preserves_operator_and_value(self):
|
||||
"""Strip only affects field name, not operator or value."""
|
||||
triples = [
|
||||
('file_id', '$in', ['a', 'b', 'c']),
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(triples, {'file_id'})
|
||||
|
||||
assert result[0] == ('file_id', '$in', ['a', 'b', 'c'])
|
||||
|
||||
def test_strip_none_aliases(self):
|
||||
"""None field_aliases is treated as empty dict."""
|
||||
triples = [
|
||||
('file_id', '$eq', 'abc'),
|
||||
]
|
||||
|
||||
result = strip_unsupported_fields(triples, {'file_id'}, field_aliases=None)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0] == ('file_id', '$eq', 'abc')
|
||||
|
||||
|
||||
class TestSupportedOpsConstant:
|
||||
"""Tests for SUPPORTED_OPS constant."""
|
||||
|
||||
def test_supported_ops_contains_expected(self):
|
||||
"""SUPPORTED_OPS contains all expected operators."""
|
||||
expected = {'$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin'}
|
||||
assert SUPPORTED_OPS == expected
|
||||
|
||||
def test_supported_ops_is_frozenset(self):
|
||||
"""SUPPORTED_OPS is a frozenset for immutability."""
|
||||
from collections.abc import Set
|
||||
|
||||
assert isinstance(SUPPORTED_OPS, Set)
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Tests for VectorDBManager provider selection logic.
|
||||
|
||||
Tests the initialization logic that selects the appropriate VDB backend
|
||||
based on configuration, without actually creating real VDB instances.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from tests.utils.import_isolation import isolated_sys_modules
|
||||
|
||||
|
||||
class TestVectorDBManagerInitialization:
|
||||
"""Tests for VectorDBManager.initialize provider selection."""
|
||||
|
||||
def _create_mock_app(self, vdb_config: dict | None):
|
||||
"""Create mock app with vdb configuration."""
|
||||
mock_app = MagicMock()
|
||||
mock_app.instance_config = MagicMock()
|
||||
mock_app.instance_config.data = MagicMock()
|
||||
mock_app.instance_config.data.get = MagicMock(return_value=vdb_config)
|
||||
mock_app.logger = MagicMock()
|
||||
mock_app.logger.info = MagicMock()
|
||||
mock_app.logger.warning = MagicMock()
|
||||
return mock_app
|
||||
|
||||
def _make_vector_import_mocks(self):
|
||||
"""Create mocks for VDB backends to prevent real imports."""
|
||||
mocks = {}
|
||||
|
||||
# Mock core.app to break circular import
|
||||
mocks['langbot.pkg.core.app'] = MagicMock()
|
||||
|
||||
# Mock all VDB backend implementations
|
||||
for backend in ['chroma', 'qdrant', 'seekdb', 'milvus', 'pgvector_db', 'valkey_search']:
|
||||
mocks[f'langbot.pkg.vector.vdbs.{backend}'] = MagicMock()
|
||||
|
||||
return mocks
|
||||
|
||||
def test_initialize_no_config_defaults_to_chroma(self):
|
||||
"""No vdb config defaults to Chroma."""
|
||||
mock_app = self._create_mock_app(None)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
# Create mock Chroma class
|
||||
mock_chroma_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.chroma'].ChromaVectorDatabase = mock_chroma_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
# Import after mocking
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
# Run initialize synchronously for test
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
# Chroma should be instantiated
|
||||
mock_chroma_class.assert_called_once_with(mock_app)
|
||||
mock_app.logger.warning.assert_called()
|
||||
|
||||
def test_initialize_chroma_backend(self):
|
||||
"""Explicit chroma config uses Chroma backend."""
|
||||
vdb_config = {'use': 'chroma'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_chroma_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.chroma'].ChromaVectorDatabase = mock_chroma_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_chroma_class.assert_called_once_with(mock_app)
|
||||
mock_app.logger.info.assert_called()
|
||||
|
||||
def test_initialize_qdrant_backend(self):
|
||||
"""Qdrant config uses Qdrant backend."""
|
||||
vdb_config = {'use': 'qdrant'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_qdrant_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.qdrant'].QdrantVectorDatabase = mock_qdrant_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_qdrant_class.assert_called_once_with(mock_app)
|
||||
|
||||
def test_initialize_seekdb_backend(self):
|
||||
"""SeekDB config uses SeekDB backend."""
|
||||
vdb_config = {'use': 'seekdb'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_seekdb_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.seekdb'].SeekDBVectorDatabase = mock_seekdb_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_seekdb_class.assert_called_once_with(mock_app)
|
||||
|
||||
def test_initialize_valkey_search_backend(self):
|
||||
"""Valkey Search config uses ValkeySearchVectorDatabase backend."""
|
||||
vdb_config = {'use': 'valkey_search'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_valkey_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.valkey_search'].ValkeySearchVectorDatabase = mock_valkey_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_valkey_class.assert_called_once_with(mock_app)
|
||||
|
||||
def test_initialize_milvus_backend_with_uri(self):
|
||||
"""Milvus config with custom URI."""
|
||||
vdb_config = {
|
||||
'use': 'milvus',
|
||||
'milvus': {'uri': 'http://localhost:19530', 'token': 'root:Milvus', 'db_name': 'langbot_db'},
|
||||
}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_milvus_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.milvus'].MilvusVectorDatabase = mock_milvus_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_milvus_class.assert_called_once_with(
|
||||
mock_app, uri='http://localhost:19530', token='root:Milvus', db_name='langbot_db'
|
||||
)
|
||||
|
||||
def test_initialize_milvus_backend_defaults(self):
|
||||
"""Milvus defaults when config not fully specified."""
|
||||
vdb_config = {'use': 'milvus'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_milvus_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.milvus'].MilvusVectorDatabase = mock_milvus_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
# Should use default values
|
||||
mock_milvus_class.assert_called_once_with(mock_app, uri='./data/milvus.db', token=None, db_name='default')
|
||||
|
||||
def test_initialize_pgvector_with_connection_string(self):
|
||||
"""pgvector with connection string."""
|
||||
vdb_config = {'use': 'pgvector', 'pgvector': {'connection_string': 'postgresql://user:pass@host:5432/langbot'}}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_pgvector_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.pgvector_db'].PgVectorDatabase = mock_pgvector_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_pgvector_class.assert_called_once_with(
|
||||
mock_app, connection_string='postgresql://user:pass@host:5432/langbot'
|
||||
)
|
||||
|
||||
def test_initialize_pgvector_with_individual_params(self):
|
||||
"""pgvector with individual connection parameters."""
|
||||
vdb_config = {
|
||||
'use': 'pgvector',
|
||||
'pgvector': {
|
||||
'host': 'db.example.com',
|
||||
'port': 5433,
|
||||
'database': 'vectordb',
|
||||
'user': 'admin',
|
||||
'password': 'secret',
|
||||
},
|
||||
}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_pgvector_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.pgvector_db'].PgVectorDatabase = mock_pgvector_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_pgvector_class.assert_called_once_with(
|
||||
mock_app, host='db.example.com', port=5433, database='vectordb', user='admin', password='secret'
|
||||
)
|
||||
|
||||
def test_initialize_pgvector_defaults(self):
|
||||
"""pgvector defaults when no config params."""
|
||||
vdb_config = {'use': 'pgvector'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_pgvector_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.pgvector_db'].PgVectorDatabase = mock_pgvector_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_pgvector_class.assert_called_once_with(
|
||||
mock_app, host='localhost', port=5432, database='langbot', user='postgres', password='postgres'
|
||||
)
|
||||
|
||||
def test_initialize_unknown_backend_defaults_to_chroma(self):
|
||||
"""Unknown vdb type defaults to Chroma with warning."""
|
||||
vdb_config = {'use': 'unknown_backend'}
|
||||
mock_app = self._create_mock_app(vdb_config)
|
||||
|
||||
mocks = self._make_vector_import_mocks()
|
||||
mock_chroma_class = MagicMock()
|
||||
mocks['langbot.pkg.vector.vdbs.chroma'].ChromaVectorDatabase = mock_chroma_class
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(mgr.initialize())
|
||||
|
||||
mock_chroma_class.assert_called_once_with(mock_app)
|
||||
mock_app.logger.warning.assert_called()
|
||||
# Should warn about no valid backend
|
||||
warning_msg = mock_app.logger.warning.call_args[0][0]
|
||||
assert 'No valid' in warning_msg or 'defaulting' in warning_msg
|
||||
|
||||
|
||||
class TestVectorDBManagerProxies:
|
||||
"""Tests for VectorDBManager proxy methods."""
|
||||
|
||||
def test_get_supported_search_types_no_vector_db(self):
|
||||
"""get_supported_search_types returns vector when no vector_db."""
|
||||
mock_app = MagicMock()
|
||||
mock_app.instance_config = MagicMock()
|
||||
mock_app.instance_config.data = MagicMock()
|
||||
mock_app.instance_config.data.get = MagicMock(return_value=None)
|
||||
mock_app.logger = MagicMock()
|
||||
|
||||
mocks = {'langbot.pkg.core.app': MagicMock()}
|
||||
for backend in ['chroma', 'qdrant', 'seekdb', 'milvus', 'pgvector_db']:
|
||||
mocks[f'langbot.pkg.vector.vdbs.{backend}'] = MagicMock()
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
mgr.vector_db = None # Explicitly None
|
||||
|
||||
result = mgr.get_supported_search_types()
|
||||
assert result == ['vector']
|
||||
|
||||
def test_get_supported_search_types_with_vector_db(self):
|
||||
"""get_supported_search_types delegates to vector_db."""
|
||||
mock_app = MagicMock()
|
||||
|
||||
# Create mock vector_db with supported_search_types
|
||||
mock_vector_db = MagicMock()
|
||||
mock_vector_db.supported_search_types = MagicMock(
|
||||
return_value=[
|
||||
MagicMock(value='vector'),
|
||||
MagicMock(value='full_text'),
|
||||
]
|
||||
)
|
||||
|
||||
mocks = {'langbot.pkg.core.app': MagicMock()}
|
||||
for backend in ['chroma', 'qdrant', 'seekdb', 'milvus', 'pgvector_db']:
|
||||
mocks[f'langbot.pkg.vector.vdbs.{backend}'] = MagicMock()
|
||||
|
||||
with isolated_sys_modules(mocks):
|
||||
from langbot.pkg.vector.mgr import VectorDBManager
|
||||
|
||||
mgr = VectorDBManager(mock_app)
|
||||
mgr.vector_db = mock_vector_db
|
||||
|
||||
result = mgr.get_supported_search_types()
|
||||
assert result == ['vector', 'full_text']
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Unit tests for the Valkey Search VDB backend's pure helpers.
|
||||
|
||||
These tests exercise the filter-to-FT mapping, float32 packing, tag/text
|
||||
escaping, FT.SEARCH reply parsing and the import guard. They run in the fast
|
||||
CI lane and require NO running Valkey server.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import struct
|
||||
from importlib import import_module
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def get_valkey_module():
|
||||
"""Lazy import of the valkey_search backend module."""
|
||||
return import_module('langbot.pkg.vector.vdbs.valkey_search')
|
||||
|
||||
|
||||
def make_backend():
|
||||
"""Construct a backend instance without running its __init__.
|
||||
|
||||
The constructor needs a live ``ap`` + config; for pure-helper tests we
|
||||
only need a bare instance with the attributes the helpers touch.
|
||||
"""
|
||||
mod = get_valkey_module()
|
||||
backend = object.__new__(mod.ValkeySearchVectorDatabase)
|
||||
# _ensure_client serializes creation through this lock; set it here since
|
||||
# __init__ (which normally creates it) is bypassed.
|
||||
backend._client_lock = asyncio.Lock()
|
||||
return backend
|
||||
|
||||
|
||||
class TestFloat32Packing:
|
||||
"""Tests for _pack_vector little-endian float32 packing."""
|
||||
|
||||
def test_pack_round_trips(self):
|
||||
mod = get_valkey_module()
|
||||
vec = [0.1, -2.5, 3.0, 4.25]
|
||||
packed = mod.ValkeySearchVectorDatabase._pack_vector(vec)
|
||||
assert isinstance(packed, bytes)
|
||||
assert len(packed) == 4 * len(vec)
|
||||
unpacked = list(struct.unpack(f'<{len(vec)}f', packed))
|
||||
for original, restored in zip(vec, unpacked):
|
||||
assert restored == pytest.approx(original, rel=1e-6)
|
||||
|
||||
def test_pack_is_little_endian(self):
|
||||
mod = get_valkey_module()
|
||||
packed = mod.ValkeySearchVectorDatabase._pack_vector([1.0])
|
||||
assert packed == struct.pack('<f', 1.0)
|
||||
|
||||
|
||||
class TestTagEscaping:
|
||||
"""Tests for _escape_tag."""
|
||||
|
||||
def test_escapes_special_chars(self):
|
||||
mod = get_valkey_module()
|
||||
escaped = mod.ValkeySearchVectorDatabase._escape_tag('a-b c.d')
|
||||
assert '\\-' in escaped
|
||||
assert '\\ ' in escaped
|
||||
assert '\\.' in escaped
|
||||
|
||||
def test_plain_value_unchanged(self):
|
||||
mod = get_valkey_module()
|
||||
assert mod.ValkeySearchVectorDatabase._escape_tag('abc123') == 'abc123'
|
||||
|
||||
|
||||
class TestFileIdEncoding:
|
||||
"""Tests for _encode_file_id (FT-unsafe char percent-encoding)."""
|
||||
|
||||
def test_uuid_is_noop(self):
|
||||
mod = get_valkey_module()
|
||||
fid = '550e8400-e29b-41d4-a716-446655440000'
|
||||
assert mod.ValkeySearchVectorDatabase._encode_file_id(fid) == fid
|
||||
|
||||
def test_encodes_braces_star_and_percent(self):
|
||||
mod = get_valkey_module()
|
||||
enc = mod.ValkeySearchVectorDatabase._encode_file_id('a{b}c*d%e')
|
||||
# '{'=7B '}'=7D '*'=2A '%'=25
|
||||
assert enc == 'a%7Bb%7Dc%2Ad%25e'
|
||||
# No raw FT-unsafe char survives.
|
||||
assert all(ch not in enc for ch in '{}*') or '%' in enc
|
||||
|
||||
def test_encoding_is_deterministic_and_collision_safe(self):
|
||||
mod = get_valkey_module()
|
||||
enc = mod.ValkeySearchVectorDatabase._encode_file_id
|
||||
# A literal "%7B" must not collide with an encoded "{".
|
||||
assert enc('{') != enc('%7B')
|
||||
assert enc('{') == '%7B'
|
||||
assert enc('%7B') == '%257B'
|
||||
|
||||
def test_filter_encodes_unsafe_chars_in_tag_query(self):
|
||||
backend = make_backend()
|
||||
# The emitted TAG query must contain the encoded form, never raw braces.
|
||||
frag = backend._triples_to_ft({'file_id': 'x}y{z*'})
|
||||
assert '7D' in frag and '7B' in frag and '2A' in frag
|
||||
# No raw '*' from the value, and exactly one opening/closing brace (the
|
||||
# TAG-clause delimiters) — the value's own braces were encoded away.
|
||||
assert '*' not in frag
|
||||
assert frag.count('{') == 1 and frag.count('}') == 1
|
||||
assert frag.startswith('@file_id:{') and frag.endswith('}')
|
||||
|
||||
def test_filter_in_operator_encodes_each_value(self):
|
||||
backend = make_backend()
|
||||
frag = backend._triples_to_ft({'file_id': {'$in': ['a*b', 'c}d']}})
|
||||
assert '2A' in frag and '7D' in frag
|
||||
assert '*' not in frag
|
||||
|
||||
|
||||
class TestFilterToFt:
|
||||
"""Tests for _triples_to_ft filter mapping (all 8 operators)."""
|
||||
|
||||
def test_empty_filter_returns_empty_string(self):
|
||||
backend = make_backend()
|
||||
assert backend._triples_to_ft(None) == ''
|
||||
assert backend._triples_to_ft({}) == ''
|
||||
|
||||
def test_eq_tag(self):
|
||||
backend = make_backend()
|
||||
assert backend._triples_to_ft({'file_id': 'abc'}) == '@file_id:{abc}'
|
||||
|
||||
def test_explicit_eq_tag(self):
|
||||
backend = make_backend()
|
||||
assert backend._triples_to_ft({'file_id': {'$eq': 'abc'}}) == '@file_id:{abc}'
|
||||
|
||||
def test_ne_tag(self):
|
||||
backend = make_backend()
|
||||
assert backend._triples_to_ft({'file_id': {'$ne': 'abc'}}) == '-@file_id:{abc}'
|
||||
|
||||
def test_in_tag(self):
|
||||
backend = make_backend()
|
||||
assert backend._triples_to_ft({'file_id': {'$in': ['a', 'b']}}) == '@file_id:{a|b}'
|
||||
|
||||
def test_nin_tag(self):
|
||||
backend = make_backend()
|
||||
assert backend._triples_to_ft({'file_id': {'$nin': ['a', 'b']}}) == '-@file_id:{a|b}'
|
||||
|
||||
def test_numeric_range_operators(self):
|
||||
backend = make_backend()
|
||||
# file_id is the only indexed field; numeric ops still render via the
|
||||
# generic range fragment, so use file_id to keep the field supported.
|
||||
# Values are cast to float (defensive against non-numeric input and a
|
||||
# future NUMERIC field becoming an injection surface).
|
||||
assert backend._triples_to_ft({'file_id': {'$gt': 5}}) == '@file_id:[(5.0 +inf]'
|
||||
assert backend._triples_to_ft({'file_id': {'$gte': 5}}) == '@file_id:[5.0 +inf]'
|
||||
assert backend._triples_to_ft({'file_id': {'$lt': 5}}) == '@file_id:[-inf (5.0]'
|
||||
assert backend._triples_to_ft({'file_id': {'$lte': 5}}) == '@file_id:[-inf 5.0]'
|
||||
|
||||
def test_numeric_range_rejects_non_numeric(self):
|
||||
backend = make_backend()
|
||||
# A non-numeric range value fails closed rather than interpolating raw.
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
backend._triples_to_ft({'file_id': {'$gt': 'not-a-number'}})
|
||||
|
||||
def test_unsupported_field_dropped(self):
|
||||
backend = make_backend()
|
||||
# Non-indexed fields are dropped (returns empty expression).
|
||||
assert backend._triples_to_ft({'some_other_field': 'x'}) == ''
|
||||
|
||||
def test_multiple_supported_keys_anded(self):
|
||||
backend = make_backend()
|
||||
# Two conditions on the same indexed field are joined with a space (AND).
|
||||
result = backend._triples_to_ft({'file_id': {'$in': ['a', 'b']}})
|
||||
assert result == '@file_id:{a|b}'
|
||||
|
||||
|
||||
class TestTextEscaping:
|
||||
"""Tests for _escape_text full-text escaping."""
|
||||
|
||||
def test_escapes_ft_special_chars(self):
|
||||
mod = get_valkey_module()
|
||||
escaped = mod.ValkeySearchVectorDatabase._escape_text('hello@world|test')
|
||||
assert '\\@' in escaped
|
||||
assert '\\|' in escaped
|
||||
|
||||
|
||||
class TestReplyToChroma:
|
||||
"""Tests for _reply_to_chroma FT.SEARCH reply parsing."""
|
||||
|
||||
def test_parses_knn_reply(self):
|
||||
backend = make_backend()
|
||||
# glide returns [total, {key: {field: value}}]
|
||||
reply = [
|
||||
2,
|
||||
{
|
||||
b'kb:col1:id1': {
|
||||
b'distance': b'0.10',
|
||||
b'document': b'hello',
|
||||
b'metadata_json': b'{"file_id": "f1"}',
|
||||
},
|
||||
b'kb:col1:id2': {
|
||||
b'distance': b'0.25',
|
||||
b'document': b'world',
|
||||
b'metadata_json': b'{"file_id": "f2"}',
|
||||
},
|
||||
},
|
||||
]
|
||||
result = backend._reply_to_chroma('idx:col1', reply, has_distance=True)
|
||||
assert result['ids'][0] == ['id1', 'id2']
|
||||
assert result['distances'][0] == [pytest.approx(0.10), pytest.approx(0.25)]
|
||||
assert result['metadatas'][0][0] == {'file_id': 'f1'}
|
||||
assert result['metadatas'][0][1] == {'file_id': 'f2'}
|
||||
|
||||
def test_empty_reply(self):
|
||||
backend = make_backend()
|
||||
result = backend._reply_to_chroma('idx:col1', [0, {}], has_distance=True)
|
||||
assert result == {'ids': [[]], 'metadatas': [[]], 'distances': [[]]}
|
||||
|
||||
def test_malformed_reply(self):
|
||||
backend = make_backend()
|
||||
result = backend._reply_to_chroma('idx:col1', [], has_distance=True)
|
||||
assert result == {'ids': [[]], 'metadatas': [[]], 'distances': [[]]}
|
||||
|
||||
def test_text_search_reply_no_distance(self):
|
||||
backend = make_backend()
|
||||
reply = [
|
||||
1,
|
||||
{
|
||||
b'kb:col1:id1': {
|
||||
b'document': b'hello',
|
||||
b'metadata_json': b'{"file_id": "f1"}',
|
||||
},
|
||||
},
|
||||
]
|
||||
result = backend._reply_to_chroma('idx:col1', reply, has_distance=False)
|
||||
assert result['ids'][0] == ['id1']
|
||||
assert result['distances'][0] == [0.0]
|
||||
|
||||
|
||||
class TestImportGuard:
|
||||
"""Tests for the ImportError guard when glide is unavailable."""
|
||||
|
||||
def test_constructor_raises_when_unavailable(self, monkeypatch):
|
||||
mod = get_valkey_module()
|
||||
monkeypatch.setattr(mod, 'VALKEY_SEARCH_AVAILABLE', False)
|
||||
with pytest.raises(ImportError, match='valkey-glide'):
|
||||
mod.ValkeySearchVectorDatabase(ap=None)
|
||||
|
||||
|
||||
class TestSupportedSearchTypes:
|
||||
"""Tests for supported_search_types."""
|
||||
|
||||
def test_supports_vector_full_text_hybrid(self):
|
||||
mod = get_valkey_module()
|
||||
from langbot.pkg.vector.vdb import SearchType
|
||||
|
||||
types = mod.ValkeySearchVectorDatabase.supported_search_types()
|
||||
assert SearchType.VECTOR in types
|
||||
assert SearchType.FULL_TEXT in types
|
||||
assert SearchType.HYBRID in types
|
||||
|
||||
|
||||
class TestDeleteByFilterGuard:
|
||||
"""Regression tests for the delete_by_filter mass-deletion guard.
|
||||
|
||||
A non-empty filter referencing only non-indexed fields must NOT fall back
|
||||
to match-all and wipe the whole collection: it must skip and return 0.
|
||||
"""
|
||||
|
||||
async def test_unsupported_only_filter_skips_and_returns_zero(self):
|
||||
backend = make_backend()
|
||||
# Make the client/index lookups succeed without a real server.
|
||||
backend._client = AsyncMock()
|
||||
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
|
||||
backend._ensure_client = AsyncMock(return_value=backend._client)
|
||||
backend._index_exists = AsyncMock(return_value=True)
|
||||
# _search_keys must never be reached for an unusable filter.
|
||||
backend._search_keys = AsyncMock(
|
||||
side_effect=AssertionError('_search_keys must not be called for an unusable filter')
|
||||
)
|
||||
|
||||
# Filter references only a non-indexed field -> maps to no FT conditions.
|
||||
deleted = await backend.delete_by_filter('col1', {'some_other_field': 'x'})
|
||||
|
||||
assert deleted == 0
|
||||
backend._client.delete.assert_not_called()
|
||||
|
||||
async def test_supported_filter_deletes_matching_keys(self):
|
||||
backend = make_backend()
|
||||
backend._client = AsyncMock()
|
||||
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
|
||||
backend._ensure_client = AsyncMock(return_value=backend._client)
|
||||
backend._index_exists = AsyncMock(return_value=True)
|
||||
backend._search_keys = AsyncMock(return_value=['kb:col1:id1', 'kb:col1:id2'])
|
||||
|
||||
deleted = await backend.delete_by_filter('col1', {'file_id': 'f1'})
|
||||
|
||||
assert deleted == 2
|
||||
backend._client.delete.assert_awaited_once_with(['kb:col1:id1', 'kb:col1:id2'])
|
||||
|
||||
|
||||
class TestClose:
|
||||
"""Tests for the close() teardown."""
|
||||
|
||||
async def test_close_resets_client_and_indexes(self):
|
||||
backend = make_backend()
|
||||
client = AsyncMock()
|
||||
backend._client = client
|
||||
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
|
||||
backend._ensured_indexes = {'idx:col1'}
|
||||
|
||||
await backend.close()
|
||||
|
||||
client.close.assert_awaited_once()
|
||||
assert backend._client is None
|
||||
assert backend._ensured_indexes == set()
|
||||
|
||||
async def test_close_is_noop_when_no_client(self):
|
||||
backend = make_backend()
|
||||
backend._client = None
|
||||
backend.ap = type('Ap', (), {'logger': AsyncMock()})()
|
||||
backend._ensured_indexes = set()
|
||||
# Should not raise.
|
||||
await backend.close()
|
||||
assert backend._client is None
|
||||
|
||||
|
||||
class TestCredentialsBuild:
|
||||
"""Tests for the auth-credential construction in _ensure_client."""
|
||||
|
||||
def _prep_backend(self, mod, monkeypatch, *, username, password):
|
||||
backend = make_backend()
|
||||
backend._client = None
|
||||
backend._host = 'localhost'
|
||||
backend._port = 6379
|
||||
backend._db = 0
|
||||
backend._tls = False
|
||||
backend._username = username
|
||||
backend._password = password
|
||||
backend._request_timeout = 5000
|
||||
backend._ensured_indexes = set()
|
||||
warnings: list[str] = []
|
||||
backend.ap = type(
|
||||
'Ap',
|
||||
(),
|
||||
{
|
||||
'logger': type(
|
||||
'L', (), {'info': lambda self, *a, **k: None, 'warning': lambda s, m, *a, **k: warnings.append(m)}
|
||||
)()
|
||||
},
|
||||
)()
|
||||
|
||||
created = {}
|
||||
|
||||
class _FakeClient:
|
||||
@staticmethod
|
||||
async def create(conf):
|
||||
created['conf'] = conf
|
||||
return AsyncMock()
|
||||
|
||||
cred_calls: list[dict] = []
|
||||
|
||||
def _fake_credentials(**kwargs):
|
||||
cred_calls.append(kwargs)
|
||||
return ('CRED', kwargs)
|
||||
|
||||
# These names are absent when the optional valkey-glide dependency is
|
||||
# unavailable (for example, on Windows), so allow the test doubles to
|
||||
# create them on the module.
|
||||
monkeypatch.setattr(mod, 'GlideClient', _FakeClient, raising=False)
|
||||
monkeypatch.setattr(mod, 'ServerCredentials', _fake_credentials, raising=False)
|
||||
monkeypatch.setattr(mod, 'GlideClientConfiguration', lambda **kw: kw, raising=False)
|
||||
monkeypatch.setattr(mod, 'NodeAddress', lambda *a, **k: ('node', a, k), raising=False)
|
||||
return backend, created, cred_calls, warnings
|
||||
|
||||
async def test_username_without_password_fails_closed(self, monkeypatch):
|
||||
mod = get_valkey_module()
|
||||
backend, created, cred_calls, warnings = self._prep_backend(mod, monkeypatch, username='acluser', password=None)
|
||||
|
||||
# A username without a password must fail closed rather than silently
|
||||
# connecting unauthenticated to a (potentially shared) Valkey instance.
|
||||
with pytest.raises(ValueError, match='without a password'):
|
||||
await backend._ensure_client()
|
||||
|
||||
assert cred_calls == [] # ServerCredentials NOT constructed
|
||||
assert 'conf' not in created # client never created
|
||||
|
||||
async def test_password_builds_credentials(self, monkeypatch):
|
||||
mod = get_valkey_module()
|
||||
backend, created, cred_calls, warnings = self._prep_backend(
|
||||
mod, monkeypatch, username='acluser', password='secret'
|
||||
)
|
||||
|
||||
await backend._ensure_client()
|
||||
|
||||
assert len(cred_calls) == 1
|
||||
assert cred_calls[0] == {'password': 'secret', 'username': 'acluser'}
|
||||
assert created['conf']['credentials'] == ('CRED', {'password': 'secret', 'username': 'acluser'})
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Tests for VectorDatabase base class and SearchType enum."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
import pytest
|
||||
|
||||
from langbot.pkg.vector.vdb import SearchType, VectorDatabase
|
||||
|
||||
|
||||
class TestSearchType:
|
||||
"""Tests for SearchType enum."""
|
||||
|
||||
def test_search_type_values(self):
|
||||
"""Test SearchType enum values."""
|
||||
assert SearchType.VECTOR.value == 'vector'
|
||||
assert SearchType.FULL_TEXT.value == 'full_text'
|
||||
assert SearchType.HYBRID.value == 'hybrid'
|
||||
|
||||
def test_search_type_is_string_enum(self):
|
||||
"""SearchType is a string enum."""
|
||||
assert isinstance(SearchType.VECTOR, str)
|
||||
assert SearchType.VECTOR == 'vector'
|
||||
|
||||
def test_search_type_from_string(self):
|
||||
"""Can create SearchType from string."""
|
||||
assert SearchType('vector') == SearchType.VECTOR
|
||||
assert SearchType('full_text') == SearchType.FULL_TEXT
|
||||
assert SearchType('hybrid') == SearchType.HYBRID
|
||||
|
||||
|
||||
class TestVectorDatabaseAbstractMethods:
|
||||
"""Tests for VectorDatabase abstract methods."""
|
||||
|
||||
def test_vector_database_is_abstract(self):
|
||||
"""VectorDatabase is abstract and cannot be instantiated directly."""
|
||||
with pytest.raises(TypeError):
|
||||
VectorDatabase()
|
||||
|
||||
def test_abstract_methods_required(self):
|
||||
"""Subclass must implement all abstract methods."""
|
||||
|
||||
class IncompleteVectorDB(VectorDatabase):
|
||||
pass
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
IncompleteVectorDB()
|
||||
|
||||
def test_supported_search_types_default(self):
|
||||
"""Default supported_search_types returns [VECTOR]."""
|
||||
|
||||
class MinimalVectorDB(VectorDatabase):
|
||||
async def add_embeddings(self, collection, ids, embeddings_list, metadatas, documents=None):
|
||||
pass
|
||||
|
||||
async def search(
|
||||
self,
|
||||
collection,
|
||||
query_embedding,
|
||||
k=5,
|
||||
search_type='vector',
|
||||
query_text='',
|
||||
filter=None,
|
||||
vector_weight=None,
|
||||
):
|
||||
pass
|
||||
|
||||
async def delete_by_file_id(self, collection, file_id):
|
||||
pass
|
||||
|
||||
async def delete_by_filter(self, collection, filter):
|
||||
pass
|
||||
|
||||
async def get_or_create_collection(self, collection):
|
||||
pass
|
||||
|
||||
async def delete_collection(self, collection):
|
||||
pass
|
||||
|
||||
db = MinimalVectorDB()
|
||||
assert db.supported_search_types() == [SearchType.VECTOR]
|
||||
|
||||
def test_list_by_filter_default_implementation(self):
|
||||
"""list_by_filter has default implementation returning empty."""
|
||||
|
||||
class MinimalVectorDB(VectorDatabase):
|
||||
async def add_embeddings(self, collection, ids, embeddings_list, metadatas, documents=None):
|
||||
pass
|
||||
|
||||
async def search(
|
||||
self,
|
||||
collection,
|
||||
query_embedding,
|
||||
k=5,
|
||||
search_type='vector',
|
||||
query_text='',
|
||||
filter=None,
|
||||
vector_weight=None,
|
||||
):
|
||||
pass
|
||||
|
||||
async def delete_by_file_id(self, collection, file_id):
|
||||
pass
|
||||
|
||||
async def delete_by_filter(self, collection, filter):
|
||||
pass
|
||||
|
||||
async def get_or_create_collection(self, collection):
|
||||
pass
|
||||
|
||||
async def delete_collection(self, collection):
|
||||
pass
|
||||
|
||||
db = MinimalVectorDB()
|
||||
# list_by_filter should return empty list and -1 for total
|
||||
import asyncio
|
||||
|
||||
result = asyncio.get_event_loop().run_until_complete(db.list_by_filter('test_collection'))
|
||||
assert result == ([], -1)
|
||||
|
||||
|
||||
class TestVectorDatabaseInterface:
|
||||
"""Tests for VectorDatabase interface contracts."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_vector_db(self):
|
||||
"""Create a minimal mock VectorDatabase for testing."""
|
||||
|
||||
class MockVectorDB(VectorDatabase):
|
||||
def __init__(self):
|
||||
self.add_embeddings = AsyncMock()
|
||||
self.search = AsyncMock(
|
||||
return_value={
|
||||
'ids': [['id1', 'id2']],
|
||||
'distances': [[0.1, 0.2]],
|
||||
'metadatas': [[{'key': 'val1'}, {'key': 'val2'}]],
|
||||
}
|
||||
)
|
||||
self.delete_by_file_id = AsyncMock()
|
||||
self.delete_by_filter = AsyncMock(return_value=5)
|
||||
self.get_or_create_collection = AsyncMock()
|
||||
self.delete_collection = AsyncMock()
|
||||
|
||||
async def add_embeddings(self, collection, ids, embeddings_list, metadatas, documents=None):
|
||||
pass
|
||||
|
||||
async def search(
|
||||
self,
|
||||
collection,
|
||||
query_embedding,
|
||||
k=5,
|
||||
search_type='vector',
|
||||
query_text='',
|
||||
filter=None,
|
||||
vector_weight=None,
|
||||
):
|
||||
pass
|
||||
|
||||
async def delete_by_file_id(self, collection, file_id):
|
||||
pass
|
||||
|
||||
async def delete_by_filter(self, collection, filter):
|
||||
pass
|
||||
|
||||
async def get_or_create_collection(self, collection):
|
||||
pass
|
||||
|
||||
async def delete_collection(self, collection):
|
||||
pass
|
||||
|
||||
return MockVectorDB()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_embeddings_signature(self, mock_vector_db):
|
||||
"""add_embeddings has expected signature."""
|
||||
await mock_vector_db.add_embeddings(
|
||||
collection='test',
|
||||
ids=['id1', 'id2'],
|
||||
embeddings_list=[[0.1, 0.2], [0.3, 0.4]],
|
||||
metadatas=[{'a': 1}, {'b': 2}],
|
||||
documents=['doc1', 'doc2'],
|
||||
)
|
||||
mock_vector_db.add_embeddings.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_signature(self, mock_vector_db):
|
||||
"""search has expected signature with all optional params."""
|
||||
import numpy as np
|
||||
|
||||
await mock_vector_db.search(
|
||||
collection='test',
|
||||
query_embedding=np.array([0.1, 0.2]),
|
||||
k=10,
|
||||
search_type='hybrid',
|
||||
query_text='search text',
|
||||
filter={'file_id': 'abc'},
|
||||
vector_weight=0.7,
|
||||
)
|
||||
mock_vector_db.search.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_by_filter_returns_int(self, mock_vector_db):
|
||||
"""delete_by_filter returns int count."""
|
||||
result = await mock_vector_db.delete_by_filter('test', {'file_id': 'abc'})
|
||||
assert isinstance(result, int)
|
||||
@@ -0,0 +1,369 @@
|
||||
"""Tests for VDB backend filter conversion functions.
|
||||
|
||||
Tests cover:
|
||||
- _build_qdrant_filter: Qdrant models.Filter conversion
|
||||
- _build_milvus_expr: Milvus boolean expression string conversion
|
||||
- _build_pg_conditions: PostgreSQL SQLAlchemy conditions conversion
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
|
||||
|
||||
def get_qdrant_module():
|
||||
"""Lazy import qdrant module."""
|
||||
return import_module('langbot.pkg.vector.vdbs.qdrant')
|
||||
|
||||
|
||||
def get_milvus_module():
|
||||
"""Lazy import milvus module."""
|
||||
return import_module('langbot.pkg.vector.vdbs.milvus')
|
||||
|
||||
|
||||
def get_pgvector_module():
|
||||
"""Lazy import pgvector module."""
|
||||
return import_module('langbot.pkg.vector.vdbs.pgvector_db')
|
||||
|
||||
|
||||
class TestQdrantFilterConversion:
|
||||
"""Tests for _build_qdrant_filter function."""
|
||||
|
||||
def test_empty_filter_returns_empty_must(self):
|
||||
"""Empty filter dict returns Filter with None must/must_not."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
|
||||
result = qdrant_module._build_qdrant_filter({})
|
||||
assert result.must is None
|
||||
assert result.must_not is None
|
||||
|
||||
def test_eq_operator_creates_must_condition(self):
|
||||
"""$eq operator creates FieldCondition in must list."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
from qdrant_client import models
|
||||
|
||||
result = qdrant_module._build_qdrant_filter({'file_id': 'abc'})
|
||||
|
||||
assert result.must is not None
|
||||
assert len(result.must) == 1
|
||||
condition = result.must[0]
|
||||
assert condition.key == 'file_id'
|
||||
assert isinstance(condition.match, models.MatchValue)
|
||||
assert condition.match.value == 'abc'
|
||||
|
||||
def test_ne_operator_creates_must_not_condition(self):
|
||||
"""$ne operator creates FieldCondition in must_not list."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
from qdrant_client import models
|
||||
|
||||
result = qdrant_module._build_qdrant_filter({'status': {'$ne': 'deleted'}})
|
||||
|
||||
assert result.must_not is not None
|
||||
assert len(result.must_not) == 1
|
||||
condition = result.must_not[0]
|
||||
assert condition.key == 'status'
|
||||
assert isinstance(condition.match, models.MatchValue)
|
||||
assert condition.match.value == 'deleted'
|
||||
|
||||
def test_in_operator_creates_match_any(self):
|
||||
"""$in operator creates MatchAny condition."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
from qdrant_client import models
|
||||
|
||||
result = qdrant_module._build_qdrant_filter({'file_type': {'$in': ['pdf', 'docx']}})
|
||||
|
||||
assert result.must is not None
|
||||
assert len(result.must) == 1
|
||||
condition = result.must[0]
|
||||
assert condition.key == 'file_type'
|
||||
assert isinstance(condition.match, models.MatchAny)
|
||||
assert condition.match.any == ['pdf', 'docx']
|
||||
|
||||
def test_nin_operator_creates_must_not_match_any(self):
|
||||
"""$nin operator creates MatchAny in must_not."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
from qdrant_client import models
|
||||
|
||||
result = qdrant_module._build_qdrant_filter({'status': {'$nin': ['deleted', 'archived']}})
|
||||
|
||||
assert result.must_not is not None
|
||||
assert len(result.must_not) == 1
|
||||
condition = result.must_not[0]
|
||||
assert condition.key == 'status'
|
||||
assert isinstance(condition.match, models.MatchAny)
|
||||
assert condition.match.any == ['deleted', 'archived']
|
||||
|
||||
def test_range_operators_create_range_condition(self):
|
||||
"""$gt, $gte, $lt, $lte create Range conditions."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
from qdrant_client import models
|
||||
|
||||
# Test $gt
|
||||
result = qdrant_module._build_qdrant_filter({'created_at': {'$gt': 100}})
|
||||
condition = result.must[0]
|
||||
assert isinstance(condition.range, models.Range)
|
||||
assert condition.range.gt == 100
|
||||
|
||||
# Test $gte
|
||||
result = qdrant_module._build_qdrant_filter({'created_at': {'$gte': 100}})
|
||||
condition = result.must[0]
|
||||
assert condition.range.gte == 100
|
||||
|
||||
# Test $lt
|
||||
result = qdrant_module._build_qdrant_filter({'created_at': {'$lt': 100}})
|
||||
condition = result.must[0]
|
||||
assert condition.range.lt == 100
|
||||
|
||||
# Test $lte
|
||||
result = qdrant_module._build_qdrant_filter({'created_at': {'$lte': 100}})
|
||||
condition = result.must[0]
|
||||
assert condition.range.lte == 100
|
||||
|
||||
def test_multiple_conditions_combined(self):
|
||||
"""Multiple conditions are combined in must/must_not."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
|
||||
result = qdrant_module._build_qdrant_filter(
|
||||
{
|
||||
'file_id': 'abc',
|
||||
'status': {'$ne': 'deleted'},
|
||||
'created_at': {'$gte': 100},
|
||||
}
|
||||
)
|
||||
|
||||
assert len(result.must) == 2 # file_id eq + created_at gte
|
||||
assert len(result.must_not) == 1 # status ne
|
||||
|
||||
def test_implicit_eq_handled(self):
|
||||
"""Implicit $eq (bare value) is correctly handled."""
|
||||
qdrant_module = get_qdrant_module()
|
||||
from qdrant_client import models
|
||||
|
||||
result = qdrant_module._build_qdrant_filter({'field': 'value'})
|
||||
|
||||
assert result.must is not None
|
||||
condition = result.must[0]
|
||||
assert isinstance(condition.match, models.MatchValue)
|
||||
|
||||
|
||||
class TestMilvusFilterConversion:
|
||||
"""Tests for _build_milvus_expr function.
|
||||
|
||||
NOTE: Milvus only supports fields: 'text', 'file_id', 'chunk_uuid'
|
||||
Tests use only these supported fields.
|
||||
"""
|
||||
|
||||
def test_empty_filter_returns_empty_string(self):
|
||||
"""Empty filter dict returns empty string."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({})
|
||||
assert result == ''
|
||||
|
||||
def test_eq_operator_expression(self):
|
||||
"""$eq operator creates == expression."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'file_id': 'abc'})
|
||||
assert result == 'file_id == "abc"'
|
||||
|
||||
def test_ne_operator_expression(self):
|
||||
"""$ne operator creates != expression."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'file_id': {'$ne': 'deleted'}})
|
||||
assert result == 'file_id != "deleted"'
|
||||
|
||||
def test_comparison_operators(self):
|
||||
"""$gt, $gte, $lt, $lte create comparison expressions."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$gt': 'uuid_100'}}) == 'chunk_uuid > "uuid_100"'
|
||||
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$gte': 'uuid_100'}}) == 'chunk_uuid >= "uuid_100"'
|
||||
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$lt': 'uuid_100'}}) == 'chunk_uuid < "uuid_100"'
|
||||
assert milvus_module._build_milvus_expr({'chunk_uuid': {'$lte': 'uuid_100'}}) == 'chunk_uuid <= "uuid_100"'
|
||||
|
||||
def test_in_operator_expression(self):
|
||||
"""$in operator creates in [...] expression."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'file_id': {'$in': ['pdf', 'docx']}})
|
||||
assert result == 'file_id in ["pdf", "docx"]'
|
||||
|
||||
def test_nin_operator_expression(self):
|
||||
"""$nin operator creates not in [...] expression."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'file_id': {'$nin': ['deleted', 'archived']}})
|
||||
assert result == 'file_id not in ["deleted", "archived"]'
|
||||
|
||||
def test_multiple_conditions_joined_with_and(self):
|
||||
"""Multiple conditions are joined with 'and'."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr(
|
||||
{
|
||||
'file_id': 'abc',
|
||||
'chunk_uuid': {'$ne': 'def'},
|
||||
}
|
||||
)
|
||||
assert 'and' in result
|
||||
assert 'file_id == "abc"' in result
|
||||
assert 'chunk_uuid != "def"' in result
|
||||
|
||||
def test_string_value_escaped(self):
|
||||
"""String values are properly escaped."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
# Test backslash escape
|
||||
result = milvus_module._build_milvus_expr({'file_id': 'C:\\Users\\test'})
|
||||
assert '\\\\' in result
|
||||
|
||||
# Test quote escape
|
||||
result = milvus_module._build_milvus_expr({'file_id': 'test "quoted"'})
|
||||
assert '\\"' in result
|
||||
|
||||
def test_text_field_supported(self):
|
||||
"""text field is supported."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'text': 'some text'})
|
||||
assert result == 'text == "some text"'
|
||||
|
||||
def test_milvus_literal_function(self):
|
||||
"""Test _milvus_literal helper."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
assert milvus_module._milvus_literal('string') == '"string"'
|
||||
assert milvus_module._milvus_literal(42) == '42'
|
||||
assert milvus_module._milvus_literal(3.14) == '3.14'
|
||||
|
||||
def test_unsupported_field_dropped(self):
|
||||
"""Unsupported fields are dropped (not in _MILVUS_SUPPORTED_FIELDS)."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'unknown_field': 'value'})
|
||||
assert result == ''
|
||||
|
||||
def test_uuid_alias_resolved(self):
|
||||
"""'uuid' alias is resolved to 'chunk_uuid'."""
|
||||
milvus_module = get_milvus_module()
|
||||
|
||||
result = milvus_module._build_milvus_expr({'uuid': 'abc'})
|
||||
assert result.startswith('chunk_uuid')
|
||||
# uuid substring appears in chunk_uuid which is expected
|
||||
|
||||
|
||||
class TestPgVectorFilterConversion:
|
||||
"""Tests for _build_pg_conditions function.
|
||||
|
||||
NOTE: PGVector only supports fields: 'text', 'file_id', 'chunk_uuid'
|
||||
Tests use only these supported fields.
|
||||
"""
|
||||
|
||||
def test_empty_filter_returns_empty_list(self):
|
||||
"""Empty filter dict returns empty list."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({})
|
||||
assert result == []
|
||||
|
||||
def test_eq_operator_creates_equality_condition(self):
|
||||
"""$eq operator creates SQLAlchemy == condition."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({'file_id': 'abc'})
|
||||
|
||||
assert len(result) == 1
|
||||
# Verify it's a SQLAlchemy BinaryExpression
|
||||
from sqlalchemy.sql.expression import BinaryExpression
|
||||
|
||||
assert isinstance(result[0], BinaryExpression)
|
||||
|
||||
def test_ne_operator_creates_inequality_condition(self):
|
||||
"""$ne operator creates SQLAlchemy != condition."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({'file_id': {'$ne': 'deleted'}})
|
||||
|
||||
assert len(result) == 1
|
||||
# Operator should be ne (not equals)
|
||||
assert '!=' in str(result[0]) or 'ne' in str(result[0].operator)
|
||||
|
||||
def test_comparison_operators(self):
|
||||
"""$gt, $gte, $lt, $lte create comparison conditions."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
# Test all comparison operators with supported field
|
||||
for op, expected_op in [
|
||||
('$gt', '>'),
|
||||
('$gte', '>='),
|
||||
('$lt', '<'),
|
||||
('$lte', '<='),
|
||||
]:
|
||||
result = pgvector_module._build_pg_conditions({'chunk_uuid': {op: 'uuid_100'}})
|
||||
assert len(result) == 1
|
||||
assert expected_op in str(result[0])
|
||||
|
||||
def test_in_operator_creates_in_condition(self):
|
||||
"""$in operator creates SQLAlchemy in_ condition."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({'file_id': {'$in': ['a', 'b', 'c']}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert 'IN' in str(result[0]).upper()
|
||||
|
||||
def test_nin_operator_creates_notin_condition(self):
|
||||
"""$nin operator creates SQLAlchemy notin_ condition."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({'file_id': {'$nin': ['a', 'b']}})
|
||||
|
||||
assert len(result) == 1
|
||||
assert 'NOT IN' in str(result[0]).upper()
|
||||
|
||||
def test_multiple_conditions_list(self):
|
||||
"""Multiple conditions return list of conditions."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions(
|
||||
{
|
||||
'file_id': 'abc',
|
||||
'chunk_uuid': {'$ne': 'def'},
|
||||
}
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
def test_unsupported_field_dropped(self):
|
||||
"""Unsupported fields are dropped (not in _PG_SUPPORTED_FIELDS)."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({'unknown_field': 'value'})
|
||||
assert result == []
|
||||
|
||||
def test_uuid_alias_resolved(self):
|
||||
"""'uuid' alias is resolved to 'chunk_uuid'."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions({'uuid': 'abc'})
|
||||
|
||||
assert len(result) == 1
|
||||
# Should reference chunk_uuid column
|
||||
assert 'chunk_uuid' in str(result[0])
|
||||
|
||||
def test_supported_fields_only(self):
|
||||
"""Only supported fields (text, file_id, chunk_uuid) are kept."""
|
||||
pgvector_module = get_pgvector_module()
|
||||
|
||||
result = pgvector_module._build_pg_conditions(
|
||||
{
|
||||
'text': {'$ne': ''},
|
||||
'file_id': 'abc',
|
||||
'chunk_uuid': {'$in': ['x', 'y']},
|
||||
'unsupported': 'value',
|
||||
}
|
||||
)
|
||||
|
||||
assert len(result) == 3 # Only supported fields
|
||||
Reference in New Issue
Block a user