chore: import upstream snapshot with attribution

backend/test/unit/plugins/test_dify_kb.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import pytest
+
+from yuxi.knowledge.implementations.dify import DifyKB
+
+
+class _FakeResponse:
+    def __init__(self, payload: dict):
+        self._payload = payload
+
+    def raise_for_status(self) -> None:
+        return None
+
+    def json(self) -> dict:
+        return self._payload
+
+
+class _FakeAsyncClient:
+    def __init__(self, response_payload: dict | None = None, raises: Exception | None = None, **kwargs):
+        del kwargs
+        self._response_payload = response_payload or {}
+        self._raises = raises
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+    async def post(self, url: str, json: dict, headers: dict):
+        assert "/datasets/" in url
+        assert headers.get("Authorization", "").startswith("Bearer ")
+        if self._raises:
+            raise self._raises
+        assert json["retrieval_model"]["search_method"] == "semantic_search"
+        assert json["retrieval_model"]["top_k"] == 5
+        assert json["retrieval_model"]["reranking_enable"] is False
+        assert json["retrieval_model"]["score_threshold_enabled"] is True
+        assert json["retrieval_model"]["score_threshold"] == 0.3
+        return _FakeResponse(self._response_payload)
+
+
+def test_dify_create_params_config_and_validation():
+    config = DifyKB.get_create_params_config()
+    keys = [option["key"] for option in config["options"]]
+    assert keys == ["dify_api_url", "dify_token", "dify_dataset_id"]
+    assert all(option["required"] for option in config["options"])
+
+    params = DifyKB.normalize_additional_params(
+        {
+            "dify_api_url": " https://api.dify.ai/v1 ",
+            "dify_token": " token ",
+            "dify_dataset_id": " dataset-123 ",
+        }
+    )
+    assert params == {
+        "dify_api_url": "https://api.dify.ai/v1",
+        "dify_token": "token",
+        "dify_dataset_id": "dataset-123",
+    }
+    assert "chunk_preset_id" not in params
+
+
+def test_dify_validation_rejects_missing_or_invalid_params():
+    with pytest.raises(ValueError, match="Dify 参数缺失"):
+        DifyKB.normalize_additional_params({"dify_api_url": "https://api.dify.ai/v1"})
+
+    with pytest.raises(ValueError, match="必须以 /v1 结尾"):
+        DifyKB.normalize_additional_params(
+            {
+                "dify_api_url": "https://api.dify.ai",
+                "dify_token": "token",
+                "dify_dataset_id": "dataset-123",
+            }
+        )
+
+
+@pytest.mark.asyncio
+async def test_dify_kb_aquery_maps_records(monkeypatch, tmp_path):
+    kb = DifyKB(str(tmp_path))
+    slug = "kb_test_dify"
+    kb.databases_meta[slug] = {
+        "name": "dify-kb",
+        "description": "test",
+        "kb_type": "dify",
+        "query_params": {
+            "options": {
+                "search_mode": "vector",
+                "final_top_k": 5,
+                "score_threshold_enabled": True,
+                "similarity_threshold": 0.3,
+            }
+        },
+        "metadata": {
+            "dify_api_url": "https://api.dify.ai/v1",
+            "dify_token": "token",
+            "dify_dataset_id": "dataset-123",
+        },
+    }
+
+    payload = {
+        "records": [
+            {
+                "score": 0.98,
+                "segment": {
+                    "id": "seg-1",
+                    "position": 2,
+                    "content": "hello world",
+                    "document": {"id": "doc-1", "name": "Doc One"},
+                },
+            }
+        ]
+    }
+
+    monkeypatch.setattr(
+        "yuxi.knowledge.implementations.dify.httpx.AsyncClient",
+        lambda **kwargs: _FakeAsyncClient(response_payload=payload, **kwargs),
+    )
+
+    result = await kb.aquery("hello", slug)
+    assert len(result) == 1
+    assert result[0]["content"] == "hello world"
+    assert result[0]["score"] == 0.98
+    assert result[0]["metadata"]["source"] == "Doc One"
+    assert result[0]["metadata"]["file_id"] == "doc-1"
+    assert result[0]["metadata"]["chunk_id"] == "seg-1"
+    assert result[0]["metadata"]["chunk_index"] == 2
+
+
+@pytest.mark.asyncio
+async def test_dify_kb_aquery_error_returns_empty(monkeypatch, tmp_path):
+    kb = DifyKB(str(tmp_path))
+    slug = "kb_test_dify_error"
+    kb.databases_meta[slug] = {
+        "name": "dify-kb",
+        "description": "test",
+        "kb_type": "dify",
+        "query_params": {"options": {}},
+        "metadata": {
+            "dify_api_url": "https://api.dify.ai/v1",
+            "dify_token": "token",
+            "dify_dataset_id": "dataset-123",
+        },
+    }
+
+    monkeypatch.setattr(
+        "yuxi.knowledge.implementations.dify.httpx.AsyncClient",
+        lambda **kwargs: _FakeAsyncClient(raises=RuntimeError("boom"), **kwargs),
+    )
+
+    result = await kb.aquery("hello", slug)
+    assert result == []

backend/test/unit/plugins/test_milvus_kb.py

@@ -0,0 +1,574 @@
+import types
+
+import pytest
+from pymilvus import CollectionSchema, DataType, FieldSchema, Function, FunctionType
+
+from yuxi.knowledge.base import FileStatus
+from yuxi.knowledge.chunking.ragflow_like.nlp import count_tokens
+from yuxi.knowledge.implementations.milvus import (
+    CONTENT_ANALYZER_PARAMS,
+    CONTENT_SPARSE_FIELD,
+    VECTOR_METRIC_TYPE,
+    MilvusKB,
+)
+
+
+class FakeHit:
+    def __init__(self, content: str, distance: float):
+        self.distance = distance
+        self.entity = {
+            "content": content,
+            "chunk_id": "chunk-1",
+            "file_id": "file-1",
+            "chunk_index": 0,
+        }
+
+
+class FakeCollection:
+    def __init__(self, distance: float = 0.8):
+        self.search_calls = []
+        self.hybrid_calls = []
+        self.insert_calls = []
+        self.distance = distance
+
+    def search(self, **kwargs):
+        self.search_calls.append(kwargs)
+        return [[FakeHit("BM25 result", self.distance)]]
+
+    def hybrid_search(self, **kwargs):
+        self.hybrid_calls.append(kwargs)
+        return [[FakeHit("Hybrid result", self.distance)]]
+
+    def insert(self, entities):
+        self.insert_calls.append(entities)
+
+
+def make_kb(collection: FakeCollection) -> MilvusKB:
+    kb = MilvusKB.__new__(MilvusKB)
+    kb.databases_meta = {"db": {"embedding_model_spec": "test-provider:test-embedding"}}
+    kb._get_query_params = lambda kb_id: {}
+    kb._get_embedding_function = lambda embedding_model_spec, **kwargs: lambda texts: [[0.1, 0.2] for _ in texts]
+
+    async def get_collection(kb_id: str):
+        return collection
+
+    async def hydrate_chunk_sources(kb_id: str, chunks: list[dict]) -> None:
+        for chunk in chunks:
+            chunk["metadata"]["source"] = "demo.md"
+
+    kb._get_milvus_collection = get_collection
+    kb._hydrate_chunk_sources = hydrate_chunk_sources
+    return kb
+
+
+def make_file_record(**overrides):
+    data = {
+        "file_id": "file-1",
+        "kb_id": "db",
+        "parent_id": None,
+        "filename": "demo.md",
+        "file_type": "md",
+        "path": "/tmp/demo.md",
+        "minio_url": None,
+        "markdown_file": "minio://parsed/db/file-1.md",
+        "status": FileStatus.PARSED,
+        "content_hash": None,
+        "file_size": 0,
+        "chunk_count": 0,
+        "token_count": 0,
+        "content_type": "file",
+        "processing_params": {},
+        "is_folder": False,
+        "error_message": None,
+        "created_by": None,
+        "updated_by": None,
+        "created_at": None,
+        "updated_at": None,
+        "original_filename": None,
+    }
+    data.update(overrides)
+    return types.SimpleNamespace(**data)
+
+
+class FakeKnowledgeFileRepository:
+    def __init__(self, records: dict[str, types.SimpleNamespace]):
+        self.records = records
+        self.update_calls = []
+        self.conditional_update_calls = []
+        self.deleted = []
+
+    async def get_by_file_id(self, file_id: str):
+        return self.records.get(file_id)
+
+    async def update_fields_if_status(self, *, kb_id: str, file_id: str, allowed_statuses: set[str], data: dict):
+        record = self.records.get(file_id)
+        self.conditional_update_calls.append((kb_id, file_id, set(allowed_statuses), dict(data)))
+        if record is None or record.kb_id != kb_id or record.status not in allowed_statuses:
+            return None
+        for key, value in data.items():
+            setattr(record, key, value)
+        return record
+
+    async def update_fields(self, *, file_id: str, data: dict, kb_id: str | None = None):
+        record = self.records.get(file_id)
+        if record is None or (kb_id and record.kb_id != kb_id):
+            return None
+        for key, value in data.items():
+            setattr(record, key, value)
+        self.update_calls.append((file_id, kb_id, dict(data)))
+        return record
+
+    async def get_filenames_by_file_ids(self, *, kb_id: str, file_ids: list[str]):
+        return {
+            file_id: record.filename
+            for file_id in file_ids
+            if (record := self.records.get(file_id)) is not None and record.kb_id == kb_id
+        }
+
+    async def list_file_ids_by_filename_contains(self, *, kb_id: str, filename_pattern: str, limit: int = 10_000):
+        return [
+            file_id
+            for file_id, record in self.records.items()
+            if record.kb_id == kb_id and filename_pattern.lower() in record.filename.lower()
+        ][:limit]
+
+    async def delete(self, file_id: str) -> None:
+        self.deleted.append(file_id)
+        self.records.pop(file_id, None)
+
+
+def patch_file_repository(monkeypatch, file_repo: FakeKnowledgeFileRepository) -> None:
+    monkeypatch.setattr("yuxi.repositories.knowledge_file_repository.KnowledgeFileRepository", lambda: file_repo)
+    monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeFileRepository", lambda: file_repo)
+
+
+def make_chunk(index: int, content: str = "content") -> dict:
+    return {
+        "id": f"id-{index}",
+        "chunk_id": f"chunk-{index}",
+        "file_id": "file-1",
+        "chunk_index": index,
+        "content": content,
+    }
+
+
+def test_build_chunk_pg_records_preserves_extraction_result():
+    kb = MilvusKB.__new__(MilvusKB)
+
+    records = kb._build_chunk_pg_records(
+        "db",
+        [
+            {
+                "chunk_id": "chunk-1",
+                "file_id": "file-1",
+                "chunk_index": 0,
+                "content": "content",
+                "extraction_result": {"entities": ["alpha"]},
+            }
+        ],
+    )
+
+    assert records[0]["extraction_result"] == {"entities": ["alpha"]}
+
+
+async def test_embed_and_store_chunks_batches_embedding_and_insert():
+    kb = MilvusKB.__new__(MilvusKB)
+    chunks = [make_chunk(index, content=f"text-{index}") for index in range(450)]
+    embedding_calls = []
+    store_calls = []
+
+    async def embedding_function(texts):
+        embedding_calls.append(list(texts))
+        return [[float(len(text))] for text in texts]
+
+    async def insert_chunks_to_stores(kb_id, file_id, collection, batch_chunks, embeddings, **kwargs):
+        store_calls.append(
+            {
+                "kb_id": kb_id,
+                "file_id": file_id,
+                "chunks": list(batch_chunks),
+                "embeddings": list(embeddings),
+                "kwargs": kwargs,
+            }
+        )
+
+    kb._insert_chunks_to_stores = insert_chunks_to_stores
+
+    await kb._embed_and_store_chunks(
+        "db",
+        "file-1",
+        FakeCollection(),
+        chunks,
+        embedding_function,
+        chunk_batch_size=200,
+    )
+
+    assert [len(call) for call in embedding_calls] == [200, 200, 50]
+    assert [len(call["chunks"]) for call in store_calls] == [200, 200, 50]
+    assert store_calls[0]["chunks"][0]["chunk_id"] == "chunk-0"
+    assert store_calls[1]["chunks"][0]["chunk_id"] == "chunk-200"
+    assert store_calls[2]["chunks"][0]["chunk_id"] == "chunk-400"
+    assert all(call["kwargs"] == {} for call in store_calls)
+
+
+def test_calculate_chunk_stats_counts_chunks_and_tokens():
+    kb = MilvusKB.__new__(MilvusKB)
+    chunks = [make_chunk(0, content="alpha beta"), make_chunk(1, content="中文")]
+
+    stats = kb._calculate_chunk_stats(chunks)
+
+    assert stats == {
+        "chunk_count": 2,
+        "token_count": count_tokens("alpha beta") + count_tokens("中文"),
+    }
+
+
+async def test_index_file_persists_chunk_stats(monkeypatch):
+    kb = MilvusKB.__new__(MilvusKB)
+    kb.databases_meta = {"db": {"embedding_model_spec": "test-provider:test-embedding", "metadata": {}}}
+    file_repo = FakeKnowledgeFileRepository({"file-1": make_file_record()})
+    patch_file_repository(monkeypatch, file_repo)
+    collection = FakeCollection()
+    deleted_files = []
+    store_calls = []
+    refreshed_kbs = []
+    chunks = [make_chunk(0, content="alpha beta"), make_chunk(1, content="中文")]
+
+    async def get_collection(kb_id):
+        return collection
+
+    async def read_markdown(path):
+        return "# demo"
+
+    async def embedding_function(texts):
+        return [[0.1, 0.2] for _ in texts]
+
+    async def delete_file_chunks_only(kb_id, file_id):
+        deleted_files.append((kb_id, file_id))
+
+    async def embed_and_store_chunks(kb_id, file_id, collection_arg, chunk_records, embedding_fn):
+        store_calls.append((kb_id, file_id, collection_arg, list(chunk_records), embedding_fn))
+
+    async def refresh_database_stats(kb_id):
+        refreshed_kbs.append(kb_id)
+        return {}
+
+    kb._get_milvus_collection = get_collection
+    kb._read_markdown_from_minio = read_markdown
+    kb._split_text_into_chunks = lambda text, file_id, filename, params: chunks
+    kb._get_embedding_function = lambda embedding_model_spec: embedding_function
+    kb.delete_file_chunks_only = delete_file_chunks_only
+    kb._embed_and_store_chunks = embed_and_store_chunks
+    kb.refresh_database_stats = refresh_database_stats
+
+    result = await kb.index_file("db", "file-1", operator_id="user-1", params={})
+
+    assert deleted_files == [("db", "file-1")]
+    assert len(store_calls) == 1
+    assert [chunk["chunk_id"] for chunk in store_calls[0][3]] == ["chunk-0", "chunk-1"]
+    assert result["status"] == FileStatus.INDEXED
+    assert result["chunk_count"] == 2
+    assert result["token_count"] == count_tokens("alpha beta") + count_tokens("中文")
+    assert file_repo.records["file-1"].chunk_count == result["chunk_count"]
+    assert file_repo.conditional_update_calls[0][3]["status"] == FileStatus.INDEXING
+    assert file_repo.update_calls[-1][2]["status"] == FileStatus.INDEXED
+    assert refreshed_kbs == ["db"]
+
+
+async def test_delete_file_chunks_only_resets_file_stats(monkeypatch):
+    repos = []
+
+    class FakeChunkRepo:
+        def __init__(self):
+            self.delete_calls = []
+            repos.append(self)
+
+        async def count_graph_indexed_by_file_id(self, file_id):
+            return 0
+
+        async def delete_by_file_id(self, file_id):
+            self.delete_calls.append(file_id)
+            return 2
+
+    monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeChunkRepository", FakeChunkRepo)
+    file_repo = FakeKnowledgeFileRepository(
+        {"file-1": make_file_record(chunk_count=2, token_count=10, status=FileStatus.INDEXED)}
+    )
+    patch_file_repository(monkeypatch, file_repo)
+    kb = MilvusKB.__new__(MilvusKB)
+    refreshed_kbs = []
+
+    async def get_collection(kb_id):
+        return None
+
+    async def refresh_database_stats(kb_id):
+        refreshed_kbs.append(kb_id)
+        return {}
+
+    kb._get_milvus_collection = get_collection
+    kb.refresh_database_stats = refresh_database_stats
+
+    await kb.delete_file_chunks_only("db", "file-1")
+
+    assert repos[0].delete_calls == ["file-1"]
+    assert file_repo.records["file-1"].chunk_count == 0
+    assert file_repo.records["file-1"].token_count == 0
+    assert file_repo.update_calls == [("file-1", "db", {"chunk_count": 0, "token_count": 0})]
+    assert refreshed_kbs == ["db"]
+
+
+async def test_insert_chunks_to_stores_inserts_current_batch(monkeypatch):
+    repos = []
+
+    class FakeChunkRepo:
+        def __init__(self):
+            self.upsert_calls = []
+            self.delete_calls = []
+            repos.append(self)
+
+        async def batch_upsert(self, chunks):
+            self.upsert_calls.append(chunks)
+            return []
+
+        async def delete_by_file_id(self, file_id):
+            self.delete_calls.append(file_id)
+            return 0
+
+    monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeChunkRepository", FakeChunkRepo)
+    kb = MilvusKB.__new__(MilvusKB)
+    collection = FakeCollection()
+    chunks = [make_chunk(index) for index in range(3)]
+    embeddings = [[0.1, 0.2] for _ in chunks]
+
+    await kb._insert_chunks_to_stores("db", "file-1", collection, chunks, embeddings)
+
+    assert len(collection.insert_calls) == 1
+    assert collection.insert_calls[0][0] == ["id-0", "id-1", "id-2"]
+    assert collection.insert_calls[0][5] == embeddings
+    assert len(repos[0].upsert_calls) == 1
+    assert [record["chunk_id"] for record in repos[0].upsert_calls[0]] == ["chunk-0", "chunk-1", "chunk-2"]
+
+
+async def test_insert_chunks_to_stores_rolls_back_file_when_milvus_insert_fails(monkeypatch):
+    repos = []
+
+    class FakeChunkRepo:
+        def __init__(self):
+            self.upsert_calls = []
+            self.delete_calls = []
+            repos.append(self)
+
+        async def batch_upsert(self, chunks):
+            self.upsert_calls.append(chunks)
+            return []
+
+        async def delete_by_file_id(self, file_id):
+            self.delete_calls.append(file_id)
+            return 0
+
+    class FailingCollection(FakeCollection):
+        def insert(self, entities):
+            super().insert(entities)
+            raise RuntimeError("milvus boom")
+
+    monkeypatch.setattr("yuxi.knowledge.implementations.milvus.KnowledgeChunkRepository", FakeChunkRepo)
+    kb = MilvusKB.__new__(MilvusKB)
+    collection = FailingCollection()
+    milvus_delete_calls = []
+
+    async def delete_file_chunks_from_milvus(collection_arg, file_id):
+        milvus_delete_calls.append((collection_arg, file_id))
+
+    kb._delete_file_chunks_from_milvus = delete_file_chunks_from_milvus
+    chunks = [make_chunk(index) for index in range(2)]
+    embeddings = [[0.1, 0.2] for _ in chunks]
+
+    with pytest.raises(RuntimeError, match="milvus boom"):
+        await kb._insert_chunks_to_stores("db", "file-1", collection, chunks, embeddings)
+
+    assert repos[0].delete_calls == ["file-1"]
+    assert milvus_delete_calls == [(collection, "file-1")]
+
+
+async def test_update_content_uses_streaming_chunk_store(monkeypatch):
+    kb = MilvusKB.__new__(MilvusKB)
+    kb.databases_meta = {"db": {"embedding_model_spec": "test-provider:test-embedding", "metadata": {}}}
+    file_repo = FakeKnowledgeFileRepository(
+        {"file-1": make_file_record(markdown_file=None, status=FileStatus.INDEXED)}
+    )
+    patch_file_repository(monkeypatch, file_repo)
+    collection = FakeCollection()
+    refreshed_kbs = []
+    deleted_files = []
+    store_calls = []
+
+    async def get_collection(kb_id):
+        return collection
+
+    async def forbidden_embedding(texts):
+        raise AssertionError("update_content should not embed the whole file directly")
+
+    async def refresh_database_stats(kb_id):
+        refreshed_kbs.append(kb_id)
+        return {}
+
+    async def delete_file_chunks_only(kb_id, file_id):
+        deleted_files.append((kb_id, file_id))
+
+    async def embed_and_store_chunks(kb_id, file_id, collection_arg, chunks, embedding_function):
+        store_calls.append((kb_id, file_id, collection_arg, list(chunks), embedding_function))
+
+    async def parse_file(source, params):
+        return "# markdown"
+
+    kb._get_milvus_collection = get_collection
+    kb._get_embedding_function = lambda embedding_model_spec: forbidden_embedding
+    kb.refresh_database_stats = refresh_database_stats
+    kb._split_text_into_chunks = lambda text, file_id, filename, params: [make_chunk(0), make_chunk(1)]
+    kb.delete_file_chunks_only = delete_file_chunks_only
+    kb._embed_and_store_chunks = embed_and_store_chunks
+    monkeypatch.setattr("yuxi.knowledge.implementations.milvus.Parser.aparse", parse_file)
+
+    result = await kb.update_content("db", ["file-1"])
+
+    assert deleted_files == [("db", "file-1")]
+    assert len(store_calls) == 1
+    assert store_calls[0][2] is collection
+    assert [chunk["chunk_id"] for chunk in store_calls[0][3]] == ["chunk-0", "chunk-1"]
+    assert store_calls[0][4] is forbidden_embedding
+    assert result[0]["status"] == FileStatus.INDEXED
+    assert file_repo.records["file-1"].status == FileStatus.INDEXED
+    assert file_repo.update_calls[0][2]["status"] == FileStatus.INDEXING
+    assert file_repo.update_calls[-1][2]["status"] == FileStatus.INDEXED
+    assert refreshed_kbs == ["db"]
+
+
+async def test_keyword_mode_uses_milvus_bm25_search():
+    collection = FakeCollection()
+    kb = make_kb(collection)
+
+    chunks = await kb.aquery(
+        "alpha beta",
+        "db",
+        search_mode="keyword",
+        bm25_top_k=7,
+        bm25_drop_ratio_search=0.2,
+    )
+
+    assert chunks[0]["content"] == "BM25 result"
+    assert chunks[0]["bm25_score"] == 0.8
+    search_call = collection.search_calls[0]
+    assert search_call["data"] == ["alpha beta"]
+    assert search_call["anns_field"] == CONTENT_SPARSE_FIELD
+    assert search_call["param"] == {
+        "metric_type": "BM25",
+        "params": {"drop_ratio_search": 0.2},
+    }
+    assert search_call["limit"] == 7
+
+
+async def test_vector_mode_ignores_metric_type_override():
+    collection = FakeCollection()
+    kb = make_kb(collection)
+
+    chunks = await kb.aquery("vector query", "db", search_mode="vector", metric_type="L2")
+
+    assert chunks[0]["content"] == "BM25 result"
+    search_call = collection.search_calls[0]
+    assert search_call["anns_field"] == "embedding"
+    assert search_call["param"]["metric_type"] == VECTOR_METRIC_TYPE
+
+
+async def test_hybrid_mode_uses_milvus_native_hybrid_search():
+    collection = FakeCollection()
+    kb = make_kb(collection)
+
+    chunks = await kb.aquery(
+        "hybrid query",
+        "db",
+        search_mode="hybrid",
+        final_top_k=3,
+        bm25_top_k=8,
+        vector_weight=0.6,
+        bm25_weight=0.4,
+    )
+
+    assert chunks[0]["content"] == "Hybrid result"
+    assert chunks[0]["hybrid_score"] == 0.8
+    hybrid_call = collection.hybrid_calls[0]
+    assert hybrid_call["limit"] == 3
+    assert hybrid_call["rerank"]._weights == [0.6, 0.4]
+
+    vector_request, bm25_request = hybrid_call["reqs"]
+    assert vector_request.anns_field == "embedding"
+    assert vector_request.data == [[0.1, 0.2]]
+    assert vector_request.param["metric_type"] == VECTOR_METRIC_TYPE
+    assert bm25_request.anns_field == CONTENT_SPARSE_FIELD
+    assert bm25_request.data == ["hybrid query"]
+    assert bm25_request.limit == 8
+    assert bm25_request.param["metric_type"] == "BM25"
+
+
+async def test_hybrid_mode_filters_scores_below_similarity_threshold():
+    collection = FakeCollection(distance=0.1)
+    kb = make_kb(collection)
+
+    chunks = await kb.aquery(
+        "hybrid query",
+        "db",
+        search_mode="hybrid",
+        final_top_k=3,
+        similarity_threshold=0.2,
+    )
+
+    assert chunks == []
+
+
+def test_query_params_config_uses_bm25_parameters():
+    kb = MilvusKB.__new__(MilvusKB)
+
+    config = kb.get_query_params_config("db")
+
+    option_keys = {option["key"] for option in config["options"]}
+    assert "keyword_top_k" not in option_keys
+    assert "metric_type" not in option_keys
+    assert {
+        "bm25_top_k",
+        "vector_weight",
+        "bm25_weight",
+        "bm25_drop_ratio_search",
+    } <= option_keys
+
+    search_mode = next(option for option in config["options"] if option["key"] == "search_mode")
+    descriptions = {option["value"]: option["description"] for option in search_mode["options"]}
+    assert "BM25" in descriptions["keyword"]
+    assert "BM25" in descriptions["hybrid"]
+
+
+def test_collection_supports_bm25_requires_analyzed_content_sparse_field_and_function():
+    kb = MilvusKB.__new__(MilvusKB)
+    schema = CollectionSchema(
+        fields=[
+            FieldSchema(name="id", dtype=DataType.VARCHAR, max_length=100, is_primary=True),
+            FieldSchema(
+                name="content",
+                dtype=DataType.VARCHAR,
+                max_length=65535,
+                enable_analyzer=True,
+                analyzer_params=CONTENT_ANALYZER_PARAMS,
+            ),
+            FieldSchema(name=CONTENT_SPARSE_FIELD, dtype=DataType.SPARSE_FLOAT_VECTOR),
+        ],
+        functions=[
+            Function(
+                name="content_bm25",
+                input_field_names=["content"],
+                output_field_names=[CONTENT_SPARSE_FIELD],
+                function_type=FunctionType.BM25,
+            )
+        ],
+    )
+
+    collection = type("Collection", (), {"schema": schema})()
+
+    assert kb._collection_supports_bm25(collection)

backend/test/unit/plugins/test_notion_kb.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import pytest
+
+from yuxi.knowledge.implementations.notion import NOTION_DEFAULT_VERSION, NotionAPIError, NotionKB
+
+
+PAGE_ID = "page-1"
+DATA_SOURCE_ID = "ds-1"
+
+
+PAGE = {
+    "object": "page",
+    "id": PAGE_ID,
+    "url": "https://www.notion.so/page-1",
+    "created_time": "2026-01-01T00:00:00.000Z",
+    "last_edited_time": "2026-01-02T00:00:00.000Z",
+    "parent": {"type": "data_source_id", "data_source_id": DATA_SOURCE_ID},
+    "properties": {
+        "Name": {"type": "title", "title": [{"plain_text": "Reasoning Paper"}]},
+        "Abstract": {"type": "rich_text", "rich_text": [{"plain_text": "Chain of thought reasoning"}]},
+    },
+}
+
+
+BLOCKS = {
+    PAGE_ID: [
+        {
+            "object": "block",
+            "id": "block-1",
+            "type": "paragraph",
+            "has_children": False,
+            "paragraph": {"rich_text": [{"plain_text": "This page discusses reasoning models."}]},
+        },
+        {
+            "object": "block",
+            "id": "block-2",
+            "type": "heading_2",
+            "has_children": False,
+            "heading_2": {"rich_text": [{"plain_text": "Evaluation"}]},
+        },
+    ]
+}
+
+
+class _FakeNotionClient:
+    def __init__(self, token: str, notion_version: str) -> None:
+        self.token = token
+        self.notion_version = notion_version
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        return False
+
+    async def search_pages(self, query_text: str, limit: int) -> list[dict]:
+        del query_text, limit
+        return [PAGE]
+
+    async def query_data_source(self, data_source_id: str, limit: int) -> list[dict]:
+        del limit
+        assert data_source_id == DATA_SOURCE_ID
+        return [PAGE]
+
+    async def retrieve_page(self, page_id: str) -> dict:
+        assert page_id == PAGE_ID
+        return PAGE
+
+    async def retrieve_block_children(self, block_id: str, limit: int) -> list[dict]:
+        del limit
+        return BLOCKS.get(block_id, [])
+
+
+class _FailingNotionClient(_FakeNotionClient):
+    async def search_pages(self, query_text: str, limit: int) -> list[dict]:
+        del query_text, limit
+        raise NotionAPIError("boom")
+
+
+class _UnknownParentNotionClient(_FakeNotionClient):
+    async def retrieve_page(self, page_id: str) -> dict:
+        page = await super().retrieve_page(page_id)
+        return {**page, "parent": {"type": "page_id", "page_id": "other-page"}}
+
+
+@pytest.fixture
+def notion_kb(tmp_path):
+    kb = NotionKB(str(tmp_path))
+    kb.databases_meta["kb_notion"] = {
+        "name": "notion-kb",
+        "description": "test",
+        "kb_type": "notion",
+        "query_params": {"options": {}},
+        "metadata": {
+            "notion_token": "token",
+            "notion_data_source_id": DATA_SOURCE_ID,
+            "notion_version": NOTION_DEFAULT_VERSION,
+        },
+    }
+    return kb
+
+
+def test_notion_create_params_config_and_validation(monkeypatch):
+    monkeypatch.delenv("NOTION_TOKEN", raising=False)
+    monkeypatch.delenv("NOTION_API_KEY", raising=False)
+
+    config = NotionKB.get_create_params_config()
+    keys = [option["key"] for option in config["options"]]
+    assert keys == ["notion_token", "notion_data_source_id", "notion_version"]
+
+    params = NotionKB.normalize_additional_params(
+        {
+            "notion_token": " token ",
+            "notion_data_source_id": " ds-1 ",
+            "notion_version": " 2026-03-11 ",
+        }
+    )
+    assert params == {
+        "notion_token": "token",
+        "notion_data_source_id": DATA_SOURCE_ID,
+        "notion_version": NOTION_DEFAULT_VERSION,
+    }
+    assert "chunk_preset_id" not in params
+
+
+def test_notion_validation_accepts_env_token(monkeypatch):
+    monkeypatch.setenv("NOTION_TOKEN", "env-token")
+    params = NotionKB.normalize_additional_params({"notion_data_source_id": DATA_SOURCE_ID})
+    assert params["notion_token"] == ""
+    assert params["notion_data_source_id"] == DATA_SOURCE_ID
+
+
+def test_notion_validation_rejects_missing_params(monkeypatch):
+    monkeypatch.delenv("NOTION_TOKEN", raising=False)
+    monkeypatch.delenv("NOTION_API_KEY", raising=False)
+
+    with pytest.raises(ValueError, match="notion_data_source_id"):
+        NotionKB.normalize_additional_params({"notion_token": "token"})
+
+    with pytest.raises(ValueError, match="notion_token"):
+        NotionKB.normalize_additional_params({"notion_data_source_id": DATA_SOURCE_ID})
+
+
+@pytest.mark.asyncio
+async def test_notion_kb_aquery_maps_pages(monkeypatch, notion_kb):
+    monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FakeNotionClient)
+
+    result = await notion_kb.aquery("reasoning", "kb_notion")
+
+    assert len(result) == 1
+    assert "reasoning" in result[0]["content"].lower()
+    assert result[0]["score"] > 0
+    assert result[0]["metadata"]["source"] == "Reasoning Paper"
+    assert result[0]["metadata"]["file_id"] == PAGE_ID
+    assert result[0]["metadata"]["chunk_id"].startswith(f"{PAGE_ID}:")
+    assert result[0]["metadata"]["notion_url"] == "https://www.notion.so/page-1"
+
+
+@pytest.mark.asyncio
+async def test_notion_open_file_content_uses_page_markdown(monkeypatch, notion_kb):
+    monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FakeNotionClient)
+
+    result = await notion_kb.open_file_content("kb_notion", PAGE_ID, offset=0, limit=3)
+
+    assert result["start_line"] == 1
+    assert result["end_line"] == 3
+    assert result["has_more_after"] is True
+    assert "Reasoning Paper" in result["content"]
+
+
+@pytest.mark.asyncio
+async def test_notion_find_file_content_uses_page_markdown(monkeypatch, notion_kb):
+    monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FakeNotionClient)
+
+    result = await notion_kb.find_file_content("kb_notion", PAGE_ID, ["models"], window_size=4)
+
+    assert result["match_mode"] == "keyword"
+    assert result["total_matches"] == 1
+    assert result["windows"]
+    assert "reasoning models" in result["windows"][0]["content"]
+
+
+@pytest.mark.asyncio
+async def test_notion_open_file_content_rejects_unknown_parent(monkeypatch, notion_kb):
+    monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _UnknownParentNotionClient)
+
+    with pytest.raises(ValueError, match="不属于当前 Data Source"):
+        await notion_kb.open_file_content("kb_notion", PAGE_ID)
+
+
+@pytest.mark.asyncio
+async def test_notion_kb_aquery_error_returns_empty(monkeypatch, notion_kb):
+    monkeypatch.setattr("yuxi.knowledge.implementations.notion._NotionClient", _FailingNotionClient)
+
+    result = await notion_kb.aquery("reasoning", "kb_notion")
+
+    assert result == []

backend/test/unit/plugins/test_ragflow_like_chunking.py

@@ -0,0 +1,364 @@
+from __future__ import annotations
+
+import os
+import sys
+
+sys.path.append(os.getcwd())
+
+from yuxi.knowledge.chunking.ragflow_like.dispatcher import chunk_markdown
+from yuxi.knowledge.chunking.ragflow_like.nlp import bullets_category, count_tokens
+from yuxi.knowledge.chunking.ragflow_like.utils.semantic_utils import split_sentences_chinese
+from yuxi.knowledge.chunking.ragflow_like.presets import (
+    CHUNK_ENGINE_VERSION,
+    CHUNK_PRESET_IDS,
+    CHUNK_PRESETS,
+    get_chunk_preset_options,
+    get_default_chunk_parser_config,
+    map_to_internal_parser_id,
+    resolve_chunk_processing_params,
+)
+from yuxi.knowledge.utils.kb_utils import resolve_processing_params, sanitize_processing_params
+
+
+def test_general_maps_to_naive() -> None:
+    assert map_to_internal_parser_id("general") == "naive"
+
+
+def test_resolve_chunk_processing_params_priority() -> None:
+    resolved = resolve_chunk_processing_params(
+        kb_additional_params={
+            "chunk_preset_id": "book",
+            "chunk_parser_config": {"chunk_token_num": 300, "delimiter": "\\n"},
+        },
+        file_processing_params={
+            "chunk_preset_id": "qa",
+            "chunk_parser_config": {"delimiter": "###", "overlapped_percent": 5},
+        },
+        request_params={
+            "chunk_preset_id": "laws",
+            "chunk_parser_config": {"chunk_token_num": 666},
+        },
+    )
+
+    assert resolved["chunk_preset_id"] == "laws"
+    assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION
+    assert resolved["chunk_parser_config"] == {
+        "chunk_token_num": 666,
+        "delimiter": "###",
+        "overlapped_percent": 5,
+    }
+
+
+def test_resolve_chunk_processing_params_returns_only_nested_keys() -> None:
+    resolved = resolve_chunk_processing_params(
+        kb_additional_params={"chunk_parser_config": {"chunk_token_num": 300}},
+        file_processing_params={},
+        request_params={},
+    )
+
+    assert resolved["chunk_parser_config"] == {"chunk_token_num": 300}
+    assert resolved["chunk_preset_id"] == "general"
+    assert resolved["chunk_engine_version"] == CHUNK_ENGINE_VERSION
+    assert len(resolved) == 3
+
+
+def test_qa_chunking_from_markdown_headings() -> None:
+    content = """
+# 问题一
+这是答案一。
+
+## 子问题
+这是答案二。
+""".strip()
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_1",
+        filename="faq.md",
+        processing_params={"chunk_preset_id": "qa", "chunk_parser_config": {}},
+    )
+
+    assert len(chunks) >= 1
+    assert "问题：" in chunks[0]["content"]
+    assert "回答：" in chunks[0]["content"]
+
+
+def test_chunk_records_include_reserved_position_fields() -> None:
+    content = "第一段内容。\n\n第二段内容。"
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_pos",
+        filename="pos.md",
+        processing_params={
+            "chunk_preset_id": "separator",
+            "chunk_parser_config": {"delimiter": "\\n\\n"},
+        },
+    )
+
+    assert chunks[0]["start_char_pos"] == 0
+    assert chunks[0]["end_char_pos"] == len("第一段内容。")
+    assert chunks[0]["start_token_pos"] is None
+    assert chunks[0]["end_token_pos"] is None
+    assert "start_char_pos" in chunks[1]
+
+
+def test_book_chunking_hierarchical_merge() -> None:
+    content = """
+第一章 总则
+第一节 适用范围
+本规范适用于测试场景。
+第二节 基本原则
+应当遵循最小改动原则。
+""".strip()
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_2",
+        filename="book.txt",
+        processing_params={"chunk_preset_id": "book", "chunk_parser_config": {"chunk_token_num": 256}},
+    )
+
+    assert len(chunks) >= 1
+    assert any("第一章" in ck["content"] for ck in chunks)
+
+
+def test_book_chunking_should_apply_overlength_protection() -> None:
+    content = "\n".join(
+        [
+            "第一章 总则",
+            "第一节 适用范围",
+            "超长正文" * 1200,
+            "第二节 基本原则",
+            "应当遵循最小改动原则。",
+        ]
+    )
+    max_chunk_tokens = 180
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_book_long",
+        filename="book.txt",
+        processing_params={
+            "chunk_preset_id": "book",
+            "chunk_parser_config": {"chunk_token_num": max_chunk_tokens, "delimiter": "\\n"},
+        },
+    )
+
+    assert len(chunks) > 1
+    assert max(count_tokens(ck["content"]) for ck in chunks) <= max_chunk_tokens
+
+
+def test_split_sentences_chinese_should_keep_quote_boundary() -> None:
+    text = '他说：“你好。”然后问：“你在吗？”最后结束！'
+    sentences = split_sentences_chinese(text)
+
+    assert sentences == ["他说：“你好。”", "然后问：“你在吗？”", "最后结束！"]
+
+
+def test_markdown_heading_has_higher_weight_in_bullet_category() -> None:
+    sections = [
+        "# 3.2 个人所得项目及计税、申报方式概括",
+        "一、关于季节工、临时工等费用税前扣除问题，以下规定继续执行。",
+        "二、根据现行规定，补贴收入应并入工资薪金所得。",
+        "（一）从超出国家规定比例支付的补贴，不属于免税福利费。",
+    ]
+
+    # 命中 markdown 标题模式（BULLET_PATTERN 下标 4）时，应该优先选中该组。
+    assert bullets_category(sections) == 4
+
+
+def test_mid_sentence_bullet_marker_should_not_be_treated_as_heading() -> None:
+    sections = [
+        "根据前述规则：一、这里是句中枚举，不是章节标题，不能被当成层级。",
+        "延续上文：（二）这里同样是正文中的枚举表达，不是独立标题。",
+        "## 3.4 交通补贴的个税处理",
+    ]
+    assert bullets_category(sections) == 4
+
+
+def test_chunk_preset_options_include_description() -> None:
+    options = get_chunk_preset_options()
+    assert [option["value"] for option in options] == list(CHUNK_PRESETS)
+    assert {option["value"] for option in options} == CHUNK_PRESET_IDS
+    assert all(isinstance(option.get("description"), str) and option["description"] for option in options)
+
+
+def test_chunk_preset_defaults_only_include_strategy_specific_fields() -> None:
+    for preset_id in CHUNK_PRESET_IDS:
+        assert get_default_chunk_parser_config(preset_id) == {}
+
+
+def test_laws_chunking_should_apply_overlength_protection() -> None:
+    lines = ["#### 中华人民共和国企业所得税法实施条例", "##### 微信扫一扫：分享"]
+    lines.extend(
+        [f"第{i}条 企业所得税法实施细则说明，适用于测试场景，确保条文长度足够用于验证分块策略。" for i in range(1, 260)]
+    )
+    content = "\n".join(lines)
+
+    max_chunk_tokens = 180
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_laws_long",
+        filename="laws.docx",
+        processing_params={
+            "chunk_preset_id": "laws",
+            "chunk_parser_config": {
+                "chunk_token_num": max_chunk_tokens,
+                "overlapped_percent": 20,
+                "delimiter": "\\n",
+            },
+        },
+    )
+
+    assert len(chunks) > 1
+    assert max(count_tokens(ck["content"]) for ck in chunks) <= max_chunk_tokens
+
+
+def test_laws_chunking_should_prefer_sentence_boundary_split() -> None:
+    line = "第一条 企业所得税法实施细则用于测试分块语义边界。"
+    content = line * 120
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_laws_sentence",
+        filename="laws.docx",
+        processing_params={
+            "chunk_preset_id": "laws",
+            "chunk_parser_config": {
+                "chunk_token_num": 120,
+                "overlapped_percent": 0,
+                "delimiter": "\\n",
+            },
+        },
+    )
+
+    assert len(chunks) > 1
+    for ck in chunks:
+        text = ck["content"].strip()
+        assert text
+        assert count_tokens(text) <= 120
+
+
+def test_laws_chunking_should_prefer_article_level_before_item_level() -> None:
+    content = """
+第六章 特别纳税调整
+第一百零六条 企业所得税法第三十八条规定的可以指定扣缴义务人的情形，包括：
+(一)在资金、经营、购销等方面存在直接或者间接的控制关系;
+(二)可以代表企业实施其他具有约束力的行为。
+第一百零七条 税务机关可以依法核定应纳税所得额。
+""".strip()
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_laws_article",
+        filename="laws.docx",
+        processing_params={
+            "chunk_preset_id": "laws",
+            "chunk_parser_config": {
+                "chunk_token_num": 1000,
+                "overlapped_percent": 0,
+                "delimiter": "\\n",
+            },
+        },
+    )
+
+    # 只要条下款项没有被拆成独立碎片，即可满足“条级优先”的目标。
+    target_chunks = [ck["content"] for ck in chunks if "第一百零六条" in ck["content"]]
+    assert target_chunks
+    assert any("(一)" in chunk and "(二)" in chunk for chunk in target_chunks)
+
+
+def test_laws_markdown_articles_should_not_collapse_into_chapter_chunk() -> None:
+    content = """
+## 第一章 总则
+- **第一条** 为了规范担保活动，保障债权实现，制定本法。
+- **第二条** 在借贷活动中，当事人可以依法设定担保。
+- **第三条** 担保活动应当遵循平等、自愿、公平和诚实信用原则。
+""".strip()
+
+    chunks = chunk_markdown(
+        markdown_content=content,
+        file_id="file_laws_markdown_article",
+        filename="laws.md",
+        processing_params={
+            "chunk_preset_id": "laws",
+            "chunk_parser_config": {
+                "chunk_token_num": 120,
+                "overlapped_percent": 0,
+                "delimiter": "\\n",
+            },
+        },
+    )
+
+    first_article_chunks = [ck["content"] for ck in chunks if "第一条" in ck["content"]]
+    assert first_article_chunks
+    # 条级切分时，第一条与第二条不应被合并到同一块。
+    assert all("第二条" not in chunk for chunk in first_article_chunks)
+    assert max(count_tokens(ck["content"]) for ck in chunks) <= 120
+
+
+def test_sanitize_processing_params_should_drop_non_persistent_fields() -> None:
+    sanitized = sanitize_processing_params(
+        {
+            "chunk_preset_id": "general",
+            "chunk_parser_config": {"chunk_token_num": 300},
+            "ocr_engine": "mineru_ocr",
+            "ocr_engine_config": {},
+            "auto_index": True,
+            "content_hashes": {"a.md": "hash-a"},
+            "enable_ocr": "mineru_ocr",
+            "_preprocessed_map": {"a.md": {"path": "/tmp/a.md"}},
+        }
+    )
+
+    assert sanitized == {
+        "chunk_preset_id": "general",
+        "chunk_parser_config": {"chunk_token_num": 300},
+        "ocr_engine": "mineru_ocr",
+        "ocr_engine_config": {},
+    }
+
+
+def test_resolve_processing_params_keeps_ocr_fields_and_chunk_params() -> None:
+    resolved = resolve_processing_params(
+        kb_additional_params={
+            "chunk_preset_id": "book",
+            "chunk_parser_config": {"delimiter": "\n", "chunk_token_num": 300},
+        },
+        file_processing_params={
+            "ocr_engine": "mineru_ocr",
+            "ocr_engine_config": {"backend": "pipeline"},
+            "chunk_preset_id": "qa",
+            "chunk_parser_config": {"overlapped_percent": 10},
+            "content_hashes": {"a.md": "hash-a"},
+        },
+        request_params={
+            "auto_index": True,
+            "chunk_preset_id": "laws",
+            "chunk_parser_config": {"chunk_token_num": 666},
+        },
+    )
+
+    assert resolved["ocr_engine"] == "mineru_ocr"
+    assert resolved["ocr_engine_config"] == {"backend": "pipeline"}
+    assert resolved["chunk_preset_id"] == "laws"
+    assert resolved["chunk_parser_config"] == {
+        "delimiter": "\n",
+        "chunk_token_num": 666,
+        "overlapped_percent": 10,
+    }
+    assert "content_hashes" not in resolved
+    assert "enable_ocr" not in resolved
+    assert "auto_index" not in resolved
+
+
+def test_resolve_processing_params_defaults_ocr_fields() -> None:
+    resolved = resolve_processing_params(
+        kb_additional_params={},
+        file_processing_params={"ocr_engine_config": "invalid", "enable_ocr": "mineru_ocr"},
+    )
+
+    assert resolved["ocr_engine"] == "rapid_ocr"
+    assert resolved["ocr_engine_config"] == {}
+    assert "enable_ocr" not in resolved