chore: import upstream snapshot with attribution

tests/embeddings/splitters/test_lazy_import_guard.py

@@ -0,0 +1,62 @@
+"""Boot-lightness guard for the lazy text-splitter import.
+
+``text_splitter_registry`` deliberately imports ``langchain_text_splitters``
+(which eagerly pulls sentence-transformers / torch / spaCy / nltk, ~500 MB)
+*lazily*, inside ``get_text_splitter`` — so it stays off the app-startup
+import chain (scheduler / blueprints / search engines all import
+``LibraryRAGService`` -> ``embeddings.splitters``). Importing it at boot
+added ~17 s to CI server startup and tipped the UI-test gates over.
+
+This test makes that contract explicit and enforced. The UI gates only
+fail when boot breaks *entirely*; this fails fast and deterministically the
+moment someone re-adds an eager heavy import to the startup chain — which is
+the textbook mitigation for lazy imports ("cover the lazy path with a
+test"). It must run in a fresh interpreter: inside the pytest process
+``langchain_text_splitters`` is already imported by sibling tests, so the
+assertion would be meaningless.
+"""
+
+import subprocess
+import sys
+import textwrap
+
+# allow: no-sut-import — the modules under test are imported inside the
+# subprocess driver below; the boot-lightness property only holds in a fresh
+# interpreter (sibling tests warm these modules in the pytest process).
+
+# Import the two modules that sit on the app-startup chain and must NOT drag
+# in the heavy splitter stack, then assert it stayed unimported.
+_DRIVER = textwrap.dedent(
+    """
+    import sys
+
+    import local_deep_research.embeddings.splitters.text_splitter_registry  # noqa: F401
+    import local_deep_research.research_library.services.library_rag_service  # noqa: F401
+
+    eager = [
+        m for m in ("langchain_text_splitters", "sentence_transformers")
+        if m in sys.modules
+    ]
+    if eager:
+        print(f"FAIL: app-startup import chain eagerly loaded {eager}")
+        sys.exit(1)
+    print("OK")
+    """
+)
+
+
+def test_startup_chain_does_not_eagerly_import_text_splitters():
+    """The startup import chain must not pull in langchain_text_splitters."""
+    result = subprocess.run(
+        [sys.executable, "-c", _DRIVER],
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert result.returncode == 0, (
+        "Boot-lightness regression: a module on the app-startup chain now "
+        "eagerly imports the heavy text-splitter stack. Keep the "
+        "langchain_text_splitters import lazy (inside get_text_splitter).\n"
+        f"stdout: {result.stdout}\nstderr: {result.stderr[-1500:]}"
+    )
+    assert "OK" in result.stdout

tests/embeddings/splitters/test_text_splitter_concurrency.py

@@ -0,0 +1,83 @@
+"""Concurrency regression test for ``get_text_splitter``.
+
+``get_text_splitter`` imports ``langchain_text_splitters`` lazily (to keep the
+heavy torch/sentence-transformers stack off the app-startup path). That
+package's ``__init__`` eagerly imports ~14 splitter submodules with enough
+internal cross-referencing that a *cold* import from several threads at once
+can observe a partially-initialized package and raise
+``ImportError: cannot import name ... from partially initialized module``.
+
+The RAG auto-index ``ThreadPoolExecutor`` and the per-user document scheduler
+both build ``LibraryRAGService`` (→ ``get_text_splitter``) from multiple worker
+threads, so this is reachable in production after a restart. ``get_text_splitter``
+serializes the cold import behind a module-level lock to prevent it.
+
+This must run in a *fresh* interpreter: inside the pytest process
+``langchain_text_splitters`` is already imported (warm), so the race window is
+gone. We therefore exercise it in a subprocess. Without the lock this fails
+deterministically (~2 of 16 threads raise); with it, all threads succeed.
+"""
+
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+# allow: no-sut-import — the SUT (get_text_splitter) is imported inside the
+# subprocess driver below, because the cold-import race only reproduces in a
+# fresh interpreter (it is already warm in the pytest process).
+
+# Driver run in a fresh interpreter: 16 threads, barrier-synced so they all hit
+# the cold ``langchain_text_splitters`` import simultaneously. recursive/token
+# only — both trigger the package init (the race source) without the
+# sentence-transformers model download.
+_DRIVER = textwrap.dedent(
+    """
+    import sys
+    import threading
+
+    from local_deep_research.embeddings.splitters import get_text_splitter
+
+    N = 16
+    barrier = threading.Barrier(N)
+    errors = []
+
+    def worker(i):
+        splitter_type = ("recursive", "token")[i % 2]
+        try:
+            barrier.wait()
+            get_text_splitter(
+                splitter_type, chunk_size=128, chunk_overlap=10
+            )
+        except Exception as exc:  # noqa: BLE001 - report any failure
+            errors.append(f"{type(exc).__name__}: {exc}")
+
+    threads = [threading.Thread(target=worker, args=(i,)) for i in range(N)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    if errors:
+        print(f"FAIL {len(errors)}/{N}: {errors[0]}")
+        sys.exit(1)
+    print("OK")
+    """
+)
+
+
+@pytest.mark.slow
+def test_get_text_splitter_concurrent_cold_import_is_safe():
+    """Concurrent first-calls must not race on the lazy package import."""
+    result = subprocess.run(
+        [sys.executable, "-c", _DRIVER],
+        capture_output=True,
+        text=True,
+        timeout=180,
+    )
+    assert result.returncode == 0, (
+        "Concurrent get_text_splitter cold-import raced.\n"
+        f"stdout: {result.stdout}\nstderr: {result.stderr[-2000:]}"
+    )
+    assert "OK" in result.stdout

tests/embeddings/splitters/test_text_splitter_edge_cases.py

@@ -0,0 +1,150 @@
+"""High-value edge case tests for embeddings/splitters/text_splitter_registry.py.
+
+Covers gaps: VALID_SPLITTER_TYPES structure, None vs empty separators,
+length_function verification, normalization across all types, semantic
+kwarg edge cases, and is_semantic_chunker_available return type.
+"""
+
+import pytest
+from unittest.mock import patch, MagicMock
+
+from local_deep_research.constants import DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS
+from local_deep_research.embeddings.splitters.text_splitter_registry import (
+    get_text_splitter,
+    is_semantic_chunker_available,
+    VALID_SPLITTER_TYPES,
+)
+
+
+class TestValidSplitterTypes:
+    """Validate the VALID_SPLITTER_TYPES constant."""
+
+    def test_is_a_list(self):
+        """VALID_SPLITTER_TYPES is a list (not set or tuple)."""
+        assert isinstance(VALID_SPLITTER_TYPES, list)
+
+    def test_has_exactly_four_entries(self):
+        """There are exactly 4 valid splitter types."""
+        assert len(VALID_SPLITTER_TYPES) == 4
+
+
+class TestRecursiveSplitterEdgeCases:
+    """Edge cases for recursive splitter type."""
+
+    def test_default_separators_exact_order(self):
+        """Default separators are in the exact expected order."""
+        splitter = get_text_splitter("recursive")
+        assert splitter._separators == DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS
+
+    def test_text_separators_none_uses_defaults(self):
+        """Explicitly passing text_separators=None uses defaults."""
+        splitter = get_text_splitter("recursive", text_separators=None)
+        assert splitter._separators == DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS
+
+    def test_empty_list_separators_passes_through(self):
+        """Empty list text_separators=[] is NOT replaced with defaults.
+
+        The code checks `if text_separators is None` not `if not text_separators`,
+        so an empty list should pass through. If it doesn't, the code replaces
+        it with defaults, which is also a valid design choice we verify here.
+        """
+        splitter = get_text_splitter("recursive", text_separators=[])
+        # Empty list is falsy but not None - test what the code actually does
+        # The code has `if text_separators is None:` so [] should pass through
+        # But RecursiveCharacterTextSplitter may add default separators internally
+        assert isinstance(splitter._separators, list)
+
+    def test_uses_len_as_length_function(self):
+        """RecursiveCharacterTextSplitter uses built-in len."""
+        splitter = get_text_splitter("recursive")
+        assert splitter._length_function is len
+
+
+class TestNormalizationAcrossTypes:
+    """Verify normalization (strip+lower) works for all splitter types."""
+
+    def test_token_type_uppercase(self):
+        """'TOKEN' normalizes to token splitter."""
+        splitter = get_text_splitter("TOKEN")
+        from langchain_text_splitters import TokenTextSplitter
+
+        assert isinstance(splitter, TokenTextSplitter)
+
+    def test_sentence_mixed_case_whitespace(self):
+        """'  Sentence  ' normalizes to sentence splitter.
+
+        Note: SentenceTransformersTokenTextSplitter requires downloading a model,
+        so we patch the constructor to avoid that overhead.
+        """
+        from unittest.mock import patch as _patch
+
+        # ``get_text_splitter`` imports this class lazily (function-local) so
+        # the heavy langchain_text_splitters/torch stack stays off the
+        # app-startup path — patch it at its source module, which is where
+        # the function-local ``from ... import`` resolves it.
+        with _patch(
+            "langchain_text_splitters.sentence_transformers.SentenceTransformersTokenTextSplitter"
+        ) as mock_cls:
+            mock_cls.return_value = MagicMock()
+            get_text_splitter("  Sentence  ")
+            mock_cls.assert_called_once()
+
+    def test_invalid_after_normalization(self):
+        """'  BOGUS  ' raises ValueError after normalization."""
+        with pytest.raises(ValueError, match="bogus"):
+            get_text_splitter("  BOGUS  ")
+
+
+class TestSemanticSplitterEdgeCases:
+    """Edge cases for semantic splitter type."""
+
+    def test_without_embeddings_raises(self):
+        """Semantic splitter without embeddings raises ValueError."""
+        with pytest.raises(ValueError, match="embeddings"):
+            get_text_splitter("semantic")
+
+    @patch(
+        "local_deep_research.embeddings.splitters.text_splitter_registry.SemanticChunker",
+        create=True,
+    )
+    def test_breakpoint_threshold_amount_zero_forwarded(self, mock_chunker_cls):
+        """breakpoint_threshold_amount=0 is forwarded (not treated as None)."""
+        mock_embeddings = MagicMock()
+
+        with patch(
+            "local_deep_research.embeddings.splitters.text_splitter_registry.SemanticChunker",
+            mock_chunker_cls,
+        ):
+            try:
+                get_text_splitter(
+                    "semantic",
+                    embeddings=mock_embeddings,
+                    breakpoint_threshold_amount=0,
+                )
+            except Exception:
+                pass  # SemanticChunker import may fail
+
+        # Verify that if the chunker was called, 0 was passed
+        if mock_chunker_cls.called:
+            call_kwargs = mock_chunker_cls.call_args[1]
+            assert "breakpoint_threshold_amount" in call_kwargs
+            assert call_kwargs["breakpoint_threshold_amount"] == 0
+
+
+class TestIsSemanticChunkerAvailable:
+    """Test is_semantic_chunker_available."""
+
+    def test_returns_bool(self):
+        """Return type is exactly bool, not a truthy ModuleSpec."""
+        result = is_semantic_chunker_available()
+        assert isinstance(result, bool)
+
+    @patch("importlib.util.find_spec", return_value=None)
+    def test_returns_false_when_not_installed(self, mock_find):
+        result = is_semantic_chunker_available()
+        assert result is False
+
+    @patch("importlib.util.find_spec", return_value=MagicMock())
+    def test_returns_true_when_installed(self, mock_find):
+        result = is_semantic_chunker_available()
+        assert result is True