chore: import upstream snapshot with attribution

python/tests/core/dify/conftest.py

@@ -0,0 +1,97 @@
+from types import SimpleNamespace
+
+import pytest
+
+from mirage.cache.index import RAMIndexCacheStore
+from mirage.cache.index.config import IndexEntry
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key
+
+
+def document(
+    document_id: str,
+    name: str,
+    *,
+    slug: object | None = None,
+    enabled: bool = True,
+    indexing_status: str = "completed",
+    archived: bool = False,
+    size: int | None = 123,
+) -> dict:
+    doc_metadata = []
+    if slug is not None:
+        doc_metadata = [{"name": "slug", "value": slug}]
+    data_source_detail_dict = {}
+    if size is not None:
+        data_source_detail_dict = {"upload_file": {"size": size}}
+    return {
+        "id": document_id,
+        "name": name,
+        "doc_metadata": doc_metadata,
+        "enabled": enabled,
+        "indexing_status": indexing_status,
+        "archived": archived,
+        "tokens": 9,
+        "data_source_type": "upload_file",
+        "data_source_detail_dict": data_source_detail_dict,
+        "created_at": 1716282000,
+    }
+
+
+def file_entry(document_id: str, name: str, size: int = 123) -> IndexEntry:
+    return IndexEntry(id=document_id,
+                      name=name,
+                      resource_type="file",
+                      size=size)
+
+
+def folder_entry(name: str) -> IndexEntry:
+    return IndexEntry(id=name, name=name, resource_type="folder")
+
+
+async def no_documents(config):
+    return []
+
+
+async def list_basic_documents(config):
+    return [
+        document("doc-1", "Quickstart", slug="guides/quickstart", size=333),
+        document("doc-2", "README.md", size=None),
+    ]
+
+
+async def list_nested_documents(config):
+    return [
+        document("doc-1", "Quickstart", slug="guides/quickstart", size=333),
+        document("doc-2", "Notes", slug="guides/deep/note", size=20),
+        document("doc-3", "README.md", size=10),
+    ]
+
+
+async def noop_ensure_tree(accessor, index, prefix=""):
+    return None
+
+
+@pytest.fixture
+def dify_accessor() -> SimpleNamespace:
+    return SimpleNamespace(config=SimpleNamespace(dataset_id="dataset-1",
+                                                  slug_metadata_name="slug"))
+
+
+@pytest.fixture
+def dify_index() -> RAMIndexCacheStore:
+    return RAMIndexCacheStore()
+
+
+@pytest.fixture
+def knowledge_root() -> PathSpec:
+    return PathSpec(resource_path=mount_key("/knowledge", "/knowledge"),
+                    virtual="/knowledge",
+                    directory="/knowledge")
+
+
+@pytest.fixture
+def guide_path() -> PathSpec:
+    return PathSpec.from_str_path(
+        "/knowledge/guides/quickstart",
+        mount_key("/knowledge/guides/quickstart", "/knowledge"))

python/tests/core/dify/test__client.py

@@ -0,0 +1,147 @@
+import httpx
+import pytest
+
+from mirage.core.dify import _client
+from mirage.resource.dify.config import DifyConfig
+
+
+def config() -> DifyConfig:
+    return DifyConfig(
+        api_key="secret",
+        base_url="https://dify.example/v1/",
+        dataset_id="dataset-1",
+    )
+
+
+@pytest.mark.asyncio
+async def test_list_all_documents_paginates_and_filters(httpx_mock):
+    httpx_mock.add_response(
+        json={
+            "data": [
+                {
+                    "id": "doc-1",
+                    "enabled": True,
+                    "indexing_status": "completed",
+                    "archived": False,
+                    "data_source_detail_dict": {
+                        "upload_file": {
+                            "size": 10
+                        }
+                    },
+                },
+                {
+                    "id": "doc-2",
+                    "enabled": False,
+                    "indexing_status": "completed",
+                    "archived": False,
+                },
+            ],
+            "has_more":
+            True,
+        })
+    httpx_mock.add_response(
+        json={
+            "data": [
+                {
+                    "id": "doc-3",
+                    "enabled": True,
+                    "indexing_status": "completed",
+                    "archived": False,
+                },
+                {
+                    "id": "doc-4",
+                    "enabled": True,
+                    "indexing_status": "indexing",
+                    "archived": False,
+                },
+            ],
+            "has_more":
+            False,
+        })
+
+    documents = await _client.list_all_documents(config())
+
+    assert [item["id"] for item in documents] == ["doc-1", "doc-3"]
+    assert documents[0]["data_source_detail_dict"]["upload_file"]["size"] == 10
+
+    requests = httpx_mock.get_requests()
+    assert len(requests) == 2
+    assert requests[0].headers["authorization"] == "Bearer secret"
+    assert requests[0].url.params["page"] == "1"
+    assert requests[0].url.params["limit"] == "100"
+    assert requests[1].url.params["page"] == "2"
+
+
+@pytest.mark.asyncio
+async def test_dify_get_retries_rate_limit(monkeypatch, httpx_mock):
+    sleeps: list[int] = []
+
+    async def sleep(delay: int) -> None:
+        sleeps.append(delay)
+
+    monkeypatch.setattr(_client.asyncio, "sleep", sleep)
+    httpx_mock.add_response(status_code=429, json={"message": "rate limit"})
+    httpx_mock.add_response(json={"ok": True})
+
+    payload = await _client.dify_get(config(), "/datasets/dataset-1/documents")
+
+    assert payload == {"ok": True}
+    assert sleeps == [1]
+
+
+@pytest.mark.asyncio
+async def test_dify_get_raises_http_status_errors(httpx_mock):
+    httpx_mock.add_response(status_code=401, json={"message": "unauthorized"})
+
+    with pytest.raises(httpx.HTTPStatusError) as exc_info:
+        await _client.dify_get(config(), "/datasets/dataset-1/documents")
+
+    assert exc_info.value.response.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_dify_post_sends_json_and_retries_server_error(
+        monkeypatch, httpx_mock):
+    sleeps: list[int] = []
+
+    async def sleep(delay: int) -> None:
+        sleeps.append(delay)
+
+    monkeypatch.setattr(_client.asyncio, "sleep", sleep)
+    httpx_mock.add_response(status_code=500, json={"message": "temporary"})
+    httpx_mock.add_response(json={"ok": True})
+
+    payload = await _client.dify_post(config(), "/datasets/dataset-1/retrieve",
+                                      {"query": "hello"})
+
+    assert payload == {"ok": True}
+    assert sleeps == [1]
+    requests = httpx_mock.get_requests()
+    assert len(requests) == 2
+    assert requests[0].headers["authorization"] == "Bearer secret"
+    assert requests[0].read() == b'{"query":"hello"}'
+
+
+@pytest.mark.asyncio
+async def test_get_document_segments_paginates_with_server_filters(httpx_mock):
+    httpx_mock.add_response(json={
+        "data": [{
+            "content": "first"
+        }],
+        "has_more": True,
+    })
+    httpx_mock.add_response(json={
+        "data": [{
+            "content": "second"
+        }],
+        "has_more": False,
+    })
+
+    segments = await _client.get_document_segments(config(), "doc-1")
+
+    assert [item["content"] for item in segments] == ["first", "second"]
+    requests = httpx_mock.get_requests()
+    assert requests[0].url.params["status"] == "completed"
+    assert requests[0].url.params["enabled"] == "true"
+    assert requests[0].url.params["page"] == "1"
+    assert requests[1].url.params["page"] == "2"

python/tests/core/dify/test_find.py

@@ -0,0 +1,72 @@
+import pytest
+
+from mirage.core.dify import find, stat, tree
+from mirage.types import FindType
+
+from .conftest import list_nested_documents
+
+
+async def get_detail(config, document_id):
+    sizes = {
+        "doc-1": 333,
+        "doc-2": 20,
+        "doc-3": 10,
+    }
+    return {
+        "id": document_id,
+        "updated_at": 1716285600,
+        "data_source_info": {
+            "upload_file": {
+                "size": sizes[document_id]
+            }
+        },
+    }
+
+
+@pytest.mark.asyncio
+async def test_find_name_matches_mount_root_start_path(monkeypatch,
+                                                       dify_accessor,
+                                                       dify_index,
+                                                       knowledge_root):
+    monkeypatch.setattr(tree, "list_all_documents", list_nested_documents)
+    monkeypatch.setattr(stat, "get_document_detail", get_detail)
+    results = await find.find(dify_accessor,
+                              knowledge_root,
+                              name="knowledge",
+                              index=dify_index)
+    assert results == ["/"]
+
+
+@pytest.mark.asyncio
+async def test_find_filters_name_type_size_and_depth(monkeypatch,
+                                                     dify_accessor, dify_index,
+                                                     knowledge_root):
+    monkeypatch.setattr(tree, "list_all_documents", list_nested_documents)
+    monkeypatch.setattr(stat, "get_document_detail", get_detail)
+
+    named = await find.find(dify_accessor,
+                            knowledge_root,
+                            name="quick*",
+                            index=dify_index)
+    assert named == ["/guides/quickstart"]
+
+    directories = await find.find(dify_accessor,
+                                  knowledge_root,
+                                  type=FindType.DIRECTORY,
+                                  index=dify_index)
+    assert directories == ["/", "/guides", "/guides/deep"]
+
+    large_files = await find.find(dify_accessor,
+                                  knowledge_root,
+                                  type=FindType.FILE,
+                                  min_size=100,
+                                  index=dify_index)
+    assert large_files == ["/guides/quickstart"]
+
+    deep = await find.find(dify_accessor,
+                           knowledge_root,
+                           mindepth=2,
+                           index=dify_index)
+    assert "/" not in deep
+    assert "/guides" not in deep
+    assert "/guides/deep" in deep

python/tests/core/dify/test_glob.py

@@ -0,0 +1,40 @@
+import pytest
+
+from mirage.core.dify import glob, tree
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key
+
+from .conftest import list_nested_documents
+
+
+@pytest.mark.asyncio
+async def test_resolve_glob_keeps_unresolved_non_pattern_path(
+        monkeypatch, dify_accessor, dify_index):
+    monkeypatch.setattr(tree, "list_all_documents", list_nested_documents)
+    path = PathSpec(resource_path=mount_key("/knowledge/guides", "/knowledge"),
+                    virtual="/knowledge/guides",
+                    directory="/knowledge/guides",
+                    resolved=False)
+
+    matches = await glob.resolve_glob(dify_accessor, [path], dify_index)
+
+    assert matches == [path]
+
+
+@pytest.mark.asyncio
+async def test_resolve_glob_matches_directory_pattern(monkeypatch,
+                                                      dify_accessor,
+                                                      dify_index):
+    monkeypatch.setattr(tree, "list_all_documents", list_nested_documents)
+    path = PathSpec(resource_path=mount_key("/knowledge/guides/quick*",
+                                            "/knowledge"),
+                    virtual="/knowledge/guides/quick*",
+                    directory="/knowledge/guides",
+                    pattern="quick*",
+                    resolved=False)
+
+    matches = await glob.resolve_glob(dify_accessor, [path], dify_index)
+
+    assert [item.virtual
+            for item in matches] == ["/knowledge/guides/quickstart"]
+    assert matches[0].directory == "/knowledge/guides/"

python/tests/core/dify/test_grep.py

@@ -0,0 +1,25 @@
+import pytest
+
+from mirage.core.dify import grep, read, tree
+
+from .conftest import list_basic_documents
+
+
+async def iter_pages(config, document_id):
+    yield [{"content": "Alpha"}]
+    yield [{"content": "beta"}]
+
+
+@pytest.mark.asyncio
+async def test_grep_bytes_matches_streamed_lines_and_records_reads(
+        monkeypatch, dify_accessor, dify_index, guide_path):
+    monkeypatch.setattr(tree, "list_all_documents", list_basic_documents)
+    monkeypatch.setattr(read, "iter_segment_pages", iter_pages)
+
+    output, reads = await grep.grep_bytes(dify_accessor, [guide_path],
+                                          "alpha",
+                                          dify_index,
+                                          ignore_case=True)
+
+    assert output == b"/knowledge/guides/quickstart:1:Alpha"
+    assert reads == {guide_path.virtual: b"Alpha\nbeta"}

python/tests/core/dify/test_path.py

@@ -0,0 +1,74 @@
+import pytest
+
+from mirage.cache.index.config import IndexEntry
+from mirage.core.dify import path
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key
+
+from .conftest import folder_entry, noop_ensure_tree
+
+
+@pytest.mark.asyncio
+async def test_resolve_path_finds_files_and_directories(
+        monkeypatch, dify_accessor, dify_index):
+    monkeypatch.setattr(path, "ensure_tree", noop_ensure_tree)
+    await dify_index.set_dir(
+        "/knowledge",
+        [
+            ("guides", folder_entry("guides")),
+            ("README.md",
+             IndexEntry(id="doc-1", name="README.md", resource_type="file")),
+        ],
+    )
+    await dify_index.set_dir("/knowledge/guides", [])
+
+    file_result = await path.resolve_path(
+        dify_accessor,
+        PathSpec.from_str_path("/knowledge/README.md",
+                               mount_key("/knowledge/README.md",
+                                         "/knowledge")),
+        dify_index,
+    )
+    dir_result = await path.resolve_path(
+        dify_accessor,
+        PathSpec.from_str_path("/knowledge/guides",
+                               mount_key("/knowledge/guides", "/knowledge")),
+        dify_index,
+    )
+
+    assert file_result.virtual_key == "/knowledge/README.md"
+    assert file_result.is_dir is False
+    assert file_result.entry is not None
+    assert dir_result.virtual_key == "/knowledge/guides"
+    assert dir_result.is_dir is True
+    assert dir_result.entry is not None
+    assert dir_result.entry.resource_type == "folder"
+
+
+@pytest.mark.asyncio
+async def test_resolve_path_raises_missing(monkeypatch, dify_accessor,
+                                           dify_index):
+    monkeypatch.setattr(path, "ensure_tree", noop_ensure_tree)
+    await dify_index.set_dir("/knowledge", [])
+
+    with pytest.raises(FileNotFoundError):
+        await path.resolve_path(
+            dify_accessor,
+            PathSpec.from_str_path(
+                "/knowledge/missing",
+                mount_key("/knowledge/missing", "/knowledge")),
+            dify_index,
+        )
+
+
+def test_virtual_key_for_honors_prefix_and_patterns():
+    assert path.virtual_key_for(
+        PathSpec.from_str_path(
+            "/knowledge/guides/quickstart",
+            "guides/quickstart")) == "/knowledge/guides/quickstart"
+    assert path.virtual_key_for(
+        PathSpec(resource_path=mount_key("/knowledge/guides/*.md",
+                                         "/knowledge"),
+                 virtual="/knowledge/guides/*.md",
+                 directory="/knowledge/guides",
+                 pattern="*.md")) == "/knowledge/guides"

python/tests/core/dify/test_read.py

@@ -0,0 +1,48 @@
+import pytest
+
+from mirage.core.dify import read, tree
+
+from .conftest import list_basic_documents
+
+
+async def get_segments(config, document_id):
+    return [{"content": "first"}, {"content": "second"}]
+
+
+async def iter_pages(config, document_id):
+    yield [{"content": "first"}]
+    yield [{"content": "second"}]
+
+
+@pytest.mark.asyncio
+async def test_read_bytes_uses_document_segments(monkeypatch, dify_accessor,
+                                                 dify_index, guide_path):
+    monkeypatch.setattr(tree, "list_all_documents", list_basic_documents)
+    monkeypatch.setattr(read, "get_document_segments", get_segments)
+
+    data = await read.read_bytes(dify_accessor, guide_path, dify_index)
+
+    assert data == b"first\nsecond"
+
+
+@pytest.mark.asyncio
+async def test_read_stream_separates_pages_with_single_newline(
+        monkeypatch, dify_accessor, dify_index, guide_path):
+    monkeypatch.setattr(tree, "list_all_documents", list_basic_documents)
+    monkeypatch.setattr(read, "iter_segment_pages", iter_pages)
+
+    chunks = [
+        chunk async for chunk in read.read_stream(dify_accessor, guide_path,
+                                                  dify_index)
+    ]
+
+    assert chunks == [b"first", b"\n", b"second"]
+
+
+@pytest.mark.asyncio
+async def test_read_bytes_rejects_directories(monkeypatch, dify_accessor,
+                                              dify_index, knowledge_root):
+    monkeypatch.setattr(tree, "list_all_documents", list_basic_documents)
+
+    with pytest.raises(IsADirectoryError):
+        await read.read_bytes(dify_accessor, knowledge_root, dify_index)

python/tests/core/dify/test_readdir.py

@@ -0,0 +1,24 @@
+import pytest
+
+from mirage.core.dify import readdir, tree
+
+from .conftest import list_basic_documents
+
+
+@pytest.mark.asyncio
+async def test_readdir_returns_directory_children(monkeypatch, dify_accessor,
+                                                  dify_index, knowledge_root):
+    monkeypatch.setattr(tree, "list_all_documents", list_basic_documents)
+
+    children = await readdir.readdir(dify_accessor, knowledge_root, dify_index)
+
+    assert children == ["/knowledge/README.md", "/knowledge/guides"]
+
+
+@pytest.mark.asyncio
+async def test_readdir_rejects_files(monkeypatch, dify_accessor, dify_index,
+                                     guide_path):
+    monkeypatch.setattr(tree, "list_all_documents", list_basic_documents)
+
+    with pytest.raises(NotADirectoryError):
+        await readdir.readdir(dify_accessor, guide_path, dify_index)

python/tests/core/dify/test_search.py

@@ -0,0 +1,428 @@
+from types import SimpleNamespace
+
+import pytest
+
+from mirage.cache.index import RAMIndexCacheStore
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key
+
+
+def document(document_id: str, name: str, slug: str | None = None) -> dict:
+    metadata = []
+    if slug is not None:
+        metadata = [{"name": "slug", "value": slug}]
+    return {
+        "id": document_id,
+        "name": name,
+        "doc_metadata": metadata,
+        "enabled": True,
+        "indexing_status": "completed",
+        "archived": False,
+        "tokens": 4,
+        "data_source_type": "upload_file",
+        "data_source_detail_dict": {
+            "upload_file": {
+                "size": 12
+            }
+        },
+        "created_at": 1716282000,
+    }
+
+
+def accessor() -> SimpleNamespace:
+    return SimpleNamespace(config=SimpleNamespace(dataset_id="dataset-1",
+                                                  slug_metadata_name="slug"))
+
+
+def custom_slug_accessor() -> SimpleNamespace:
+    return SimpleNamespace(config=SimpleNamespace(dataset_id="dataset-1",
+                                                  slug_metadata_name="path"))
+
+
+custom_slug_search_bodies: list[dict] = []
+
+
+async def list_path_metadata_documents(config):
+    return [{
+        **document("doc-1", "API"), "doc_metadata": [{
+            "name": "path",
+            "value": "guides/api"
+        }]
+    }]
+
+
+async def record_empty_custom_slug_search(config, endpoint, body):
+    custom_slug_search_bodies.append(body)
+    return {"records": []}
+
+
+@pytest.mark.asyncio
+async def test_search_segments_scopes_folder_to_all_documents(monkeypatch):
+    from mirage.core.dify import search, tree
+
+    bodies: list[dict] = []
+
+    async def list_documents(config):
+        return [
+            document("doc-1", "API", "guides/api"),
+            document("doc-2", "Auth", "guides/auth"),
+            document("doc-3", "README.md"),
+        ]
+
+    async def dify_post(config, endpoint, body):
+        bodies.append(body)
+        return {
+            "records": [{
+                "segment": {
+                    "content": "api segment",
+                    "document": {
+                        "name": "API",
+                        "doc_metadata": [{
+                            "name": "slug",
+                            "value": "guides/api"
+                        }],
+                    },
+                },
+                "score": 0.92,
+            }, {
+                "segment": {
+                    "content": "auth segment",
+                    "document": {
+                        "name": "Auth",
+                        "doc_metadata": [{
+                            "name": "slug",
+                            "value": "guides/auth"
+                        }],
+                    },
+                },
+                "score": 0.81,
+            }]
+        }
+
+    monkeypatch.setattr(tree, "list_all_documents", list_documents)
+    monkeypatch.setattr(search, "dify_post", dify_post)
+
+    result = await search.search_segments(
+        accessor(),
+        "login docs",
+        [
+            PathSpec(resource_path=mount_key("/knowledge/guides",
+                                             "/knowledge"),
+                     virtual="/knowledge/guides",
+                     directory="/knowledge/guides")
+        ],
+        RAMIndexCacheStore(),
+        method="hybrid",
+        top_k=2,
+        threshold=0.4,
+    )
+
+    assert result == (b"/knowledge/guides/api:0.92\napi segment\n"
+                      b"/knowledge/guides/auth:0.81\nauth segment\n")
+    retrieval = bodies[0]["retrieval_model"]
+    assert retrieval["search_method"] == "hybrid_search"
+    assert retrieval["top_k"] == 2
+    assert retrieval["score_threshold_enabled"] is True
+    assert retrieval["score_threshold"] == 0.4
+    assert retrieval["metadata_filtering_conditions"] == {
+        "logical_operator":
+        "or",
+        "conditions": [{
+            "name": "slug",
+            "comparison_operator": "in",
+            "value": ["guides/api", "guides/auth"],
+        }],
+    }
+
+
+@pytest.mark.asyncio
+async def test_search_segments_unions_slug_and_name_based_paths(monkeypatch):
+    from mirage.core.dify import search, tree
+
+    bodies: list[dict] = []
+
+    async def list_documents(config):
+        return [
+            document("doc-1", "API", "guides/api"),
+            document("doc-2", "README.md"),
+        ]
+
+    async def dify_post(config, endpoint, body):
+        bodies.append(body)
+        return {"records": []}
+
+    monkeypatch.setattr(tree, "list_all_documents", list_documents)
+    monkeypatch.setattr(search, "dify_post", dify_post)
+
+    result = await search.search_segments(
+        accessor(),
+        "setup",
+        [
+            PathSpec(resource_path=mount_key("/knowledge/guides/api",
+                                             "/knowledge"),
+                     virtual="/knowledge/guides/api",
+                     directory="/knowledge/guides/api"),
+            PathSpec(resource_path=mount_key("/knowledge/README.md",
+                                             "/knowledge"),
+                     virtual="/knowledge/README.md",
+                     directory="/knowledge/README.md"),
+        ],
+        RAMIndexCacheStore(),
+    )
+
+    assert result == b""
+    assert bodies[0]["retrieval_model"]["metadata_filtering_conditions"] == {
+        "logical_operator":
+        "or",
+        "conditions": [{
+            "name": "slug",
+            "comparison_operator": "in",
+            "value": ["guides/api"],
+        }, {
+            "name": "document_name",
+            "comparison_operator": "in",
+            "value": ["README.md"],
+        }],
+    }
+
+
+@pytest.mark.asyncio
+async def test_search_segments_uses_configured_slug_metadata_name(monkeypatch):
+    from mirage.core.dify import search, tree
+
+    custom_slug_search_bodies.clear()
+    monkeypatch.setattr(tree, "list_all_documents",
+                        list_path_metadata_documents)
+    monkeypatch.setattr(search, "dify_post", record_empty_custom_slug_search)
+
+    await search.search_segments(
+        custom_slug_accessor(),
+        "setup",
+        [
+            PathSpec(resource_path=mount_key("/knowledge/guides/api",
+                                             "/knowledge"),
+                     virtual="/knowledge/guides/api",
+                     directory="/knowledge/guides/api")
+        ],
+        RAMIndexCacheStore(),
+    )
+
+    assert custom_slug_search_bodies[0]["retrieval_model"][
+        "metadata_filtering_conditions"] == {
+            "logical_operator":
+            "or",
+            "conditions": [{
+                "name": "path",
+                "comparison_operator": "in",
+                "value": ["guides/api"],
+            }],
+        }
+
+
+@pytest.mark.asyncio
+async def test_search_segments_empty_paths_searches_whole_dataset(monkeypatch):
+    from mirage.core.dify import search
+
+    bodies: list[dict] = []
+
+    async def dify_post(config, endpoint, body):
+        bodies.append(body)
+        return {
+            "records": [{
+                "segment": {
+                    "content": "dataset match",
+                    "document": {
+                        "name": "README.md",
+                        "doc_metadata": None,
+                    },
+                },
+                "score": 0.77,
+            }]
+        }
+
+    monkeypatch.setattr(search, "dify_post", dify_post)
+
+    result = await search.search_segments(accessor(),
+                                          "anything", [],
+                                          RAMIndexCacheStore(),
+                                          mount_prefix="/knowledge/")
+
+    assert result == b"/knowledge/README.md:0.77\ndataset match\n"
+    assert "metadata_filtering_conditions" not in bodies[0]["retrieval_model"]
+
+
+def test_records_to_bytes_formats_absolute_paths_and_scores():
+    from mirage.core.dify.search import records_to_bytes
+
+    result = records_to_bytes(
+        [{
+            "segment": {
+                "content": "api segment",
+                "document": {
+                    "name": "API",
+                    "doc_metadata": [{
+                        "name": "slug",
+                        "value": "guides/api"
+                    }],
+                },
+            },
+            "score": 0.92,
+        }, {
+            "segment": {
+                "content": "auth segment",
+                "document": {
+                    "name": "README.md",
+                    "doc_metadata": None,
+                },
+            },
+        }],
+        "slug",
+        "/knowledge/",
+    )
+
+    assert result == (b"/knowledge/guides/api:0.92\napi segment\n"
+                      b"/knowledge/README.md\nauth segment\n")
+
+
+def test_records_to_bytes_keeps_multiple_chunks_for_same_document():
+    from mirage.core.dify.search import records_to_bytes
+
+    result = records_to_bytes(
+        [{
+            "segment": {
+                "content": "first chunk",
+                "document": {
+                    "name":
+                    "refunds.md",
+                    "doc_metadata": [{
+                        "name": "slug",
+                        "value": "policies/refunds"
+                    }],
+                },
+            },
+            "score": 0.82,
+        }, {
+            "segment": {
+                "content": "second chunk",
+                "document": {
+                    "name":
+                    "refunds.md",
+                    "doc_metadata": [{
+                        "name": "slug",
+                        "value": "policies/refunds"
+                    }],
+                },
+            },
+            "score": 0.79,
+        }],
+        "slug",
+        "/knowledge/",
+    )
+
+    assert result == (b"/knowledge/policies/refunds:0.82\nfirst chunk\n"
+                      b"/knowledge/policies/refunds:0.79\nsecond chunk\n")
+
+
+def test_records_to_bytes_skips_records_with_invalid_slug():
+    from mirage.core.dify.search import records_to_bytes
+
+    result = records_to_bytes(
+        [{
+            "segment": {
+                "content": "bad chunk",
+                "document": {
+                    "name": "ok.md",
+                    "doc_metadata": [{
+                        "name": "slug",
+                        "value": ""
+                    }],
+                },
+            },
+            "score": 0.82,
+        }, {
+            "segment": {
+                "content": "good chunk",
+                "document": {
+                    "name":
+                    "refunds.md",
+                    "doc_metadata": [{
+                        "name": "slug",
+                        "value": "policies/refunds"
+                    }],
+                },
+            },
+            "score": 0.79,
+        }],
+        "slug",
+        "/knowledge/",
+    )
+
+    assert result == b"/knowledge/policies/refunds:0.79\ngood chunk\n"
+
+
+def test_records_to_bytes_omits_score_for_non_numeric_values():
+    from mirage.core.dify.search import records_to_bytes
+
+    result = records_to_bytes(
+        [{
+            "segment": {
+                "content": "chunk",
+                "document": {
+                    "name":
+                    "refunds.md",
+                    "doc_metadata": [{
+                        "name": "slug",
+                        "value": "policies/refunds"
+                    }],
+                },
+            },
+            "score": True,
+        }],
+        "slug",
+        "/knowledge/",
+    )
+
+    assert result == b"/knowledge/policies/refunds\nchunk\n"
+
+
+@pytest.mark.asyncio
+async def test_search_segments_caps_top_k_at_dify_limit(monkeypatch):
+    from mirage.core.dify import search
+
+    bodies: list[dict] = []
+
+    async def dify_post(config, endpoint, body):
+        bodies.append(body)
+        return {"records": []}
+
+    monkeypatch.setattr(search, "dify_post", dify_post)
+
+    await search.search_segments(accessor(),
+                                 "anything", [],
+                                 RAMIndexCacheStore(),
+                                 top_k=150)
+
+    assert bodies[0]["retrieval_model"]["top_k"] == 100
+
+
+@pytest.mark.asyncio
+async def test_search_segments_validates_arguments():
+    from mirage.core.dify.search import search_segments
+
+    with pytest.raises(ValueError, match="search: query is required"):
+        await search_segments(accessor(), "", [], RAMIndexCacheStore())
+    with pytest.raises(ValueError, match="search: top-k must be positive"):
+        await search_segments(accessor(),
+                              "x", [],
+                              RAMIndexCacheStore(),
+                              top_k=0)
+    with pytest.raises(ValueError, match="search: threshold must be in"):
+        await search_segments(accessor(),
+                              "x", [],
+                              RAMIndexCacheStore(),
+                              threshold=1.2)
+    with pytest.raises(ValueError, match="search: method must be one of"):
+        await search_segments(accessor(),
+                              "x", [],
+                              RAMIndexCacheStore(),
+                              method="bad")

python/tests/core/dify/test_stat.py

@@ -0,0 +1,95 @@
+from types import SimpleNamespace
+
+import pytest
+
+from mirage.cache.index import RAMIndexCacheStore
+from mirage.types import FileType, PathSpec
+from mirage.utils.key_prefix import mount_key
+
+
+def document(
+    document_id: str,
+    name: str,
+    *,
+    slug: str,
+    size: int = 123,
+) -> dict:
+    return {
+        "id": document_id,
+        "name": name,
+        "doc_metadata": [{
+            "name": "slug",
+            "value": slug
+        }],
+        "enabled": True,
+        "indexing_status": "completed",
+        "archived": False,
+        "tokens": 9,
+        "data_source_type": "upload_file",
+        "data_source_detail_dict": {
+            "upload_file": {
+                "size": size
+            }
+        },
+        "created_at": 1716282000,
+    }
+
+
+def accessor() -> SimpleNamespace:
+    return SimpleNamespace(config=SimpleNamespace(slug_metadata_name="slug"))
+
+
+@pytest.mark.asyncio
+async def test_stat_light_uses_index_entry_without_detail_call(monkeypatch):
+    from mirage.core.dify import stat, tree
+
+    async def list_documents(config):
+        return [document("doc-1", "Quickstart", slug="guides/quickstart")]
+
+    async def get_detail(config, document_id):
+        raise AssertionError("stat_light should not fetch document detail")
+
+    monkeypatch.setattr(tree, "list_all_documents", list_documents)
+    monkeypatch.setattr(stat, "get_document_detail", get_detail)
+
+    index = RAMIndexCacheStore()
+    path = PathSpec(
+        resource_path=mount_key("/knowledge/guides/quickstart", "/knowledge"),
+        virtual="/knowledge/guides/quickstart",
+        directory="/knowledge/guides/quickstart",
+    )
+
+    item = await stat.stat_light(accessor(), path, index)
+
+    assert item.name == "quickstart"
+    assert item.type == FileType.TEXT
+    assert item.size == 123
+    assert item.modified == "2024-05-21T09:00:00+00:00"
+    assert item.extra["slug"] == "guides/quickstart"
+
+
+@pytest.mark.asyncio
+async def test_stat_light_returns_directory_without_detail_call(monkeypatch):
+    from mirage.core.dify import stat, tree
+
+    async def list_documents(config):
+        return [document("doc-1", "Quickstart", slug="guides/quickstart")]
+
+    async def get_detail(config, document_id):
+        raise AssertionError("stat_light should not fetch document detail")
+
+    monkeypatch.setattr(tree, "list_all_documents", list_documents)
+    monkeypatch.setattr(stat, "get_document_detail", get_detail)
+
+    index = RAMIndexCacheStore()
+    path = PathSpec(
+        resource_path=mount_key("/knowledge/guides", "/knowledge"),
+        virtual="/knowledge/guides",
+        directory="/knowledge/guides",
+    )
+
+    item = await stat.stat_light(accessor(), path, index)
+
+    assert item.name == "guides"
+    assert item.type == FileType.DIRECTORY
+    assert item.extra == {"children_count": 0}

python/tests/core/dify/test_tree.py

@@ -0,0 +1,125 @@
+import pytest
+
+from mirage.core.dify import tree
+from mirage.core.dify._client import is_visible_document
+
+from .conftest import document
+
+tree_calls = {"documents": 0}
+
+
+async def list_documents_with_hidden_and_fallbacks(config):
+    # Mirrors the list_all_documents contract: hidden documents (archived,
+    # disabled, still indexing) never leave the client.
+    documents = [
+        document("doc-1", "Quickstart", slug="guides/quickstart", size=333),
+        document("doc-2", "API", slug="api", size=444, archived=True),
+        document("doc-3", "Draft", slug="draft", indexing_status="indexing"),
+        document("doc-4", "Disabled", slug="disabled", enabled=False),
+        document("doc-5", "Archived", slug="archived", archived=True),
+        document("doc-6", "README.md", size=None),
+    ]
+    return [doc for doc in documents if is_visible_document(doc)]
+
+
+async def counted_documents(config):
+    tree_calls["documents"] += 1
+    return await list_documents_with_hidden_and_fallbacks(config)
+
+
+async def duplicate_documents(config):
+    return [
+        document("doc-1", "one", slug="same"),
+        document("doc-2", "two", slug="same"),
+    ]
+
+
+async def collision_documents(config):
+    return [
+        document("doc-1", "foo", slug="foo"),
+        document("doc-2", "bar", slug="foo/bar"),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_ensure_tree_builds_prefixed_entries_and_uses_api_size(
+        monkeypatch, dify_accessor, dify_index):
+    tree_calls["documents"] = 0
+    monkeypatch.setattr(tree, "list_all_documents", counted_documents)
+
+    await tree.ensure_tree(dify_accessor, dify_index, "/knowledge/")
+
+    root = await dify_index.list_dir("/knowledge")
+    guides = await dify_index.list_dir("/knowledge/guides")
+    quickstart = await dify_index.get("/knowledge/guides/quickstart")
+    readme = await dify_index.get("/knowledge/README.md")
+
+    assert root.entries == ["/knowledge/README.md", "/knowledge/guides"]
+    assert guides.entries == ["/knowledge/guides/quickstart"]
+    assert quickstart.entry.id == "doc-1"
+    assert quickstart.entry.size == 333
+    assert quickstart.entry.extra["slug"] == "guides/quickstart"
+    assert quickstart.entry.extra["raw_slug"] == "guides/quickstart"
+    assert quickstart.entry.extra["has_slug"] is True
+    assert readme.entry.id == "doc-6"
+    assert readme.entry.size is None
+    assert readme.entry.extra["raw_slug"] == "README.md"
+    assert readme.entry.extra["has_slug"] is False
+
+    await tree.ensure_tree(dify_accessor, dify_index, "/knowledge/")
+    assert tree_calls["documents"] == 1
+
+
+@pytest.mark.asyncio
+async def test_ensure_tree_rejects_duplicate_and_path_collision(
+        monkeypatch, dify_accessor, dify_index):
+    monkeypatch.setattr(tree, "list_all_documents", duplicate_documents)
+    with pytest.raises(ValueError, match="Duplicate slug 'same'"):
+        await tree.ensure_tree(dify_accessor, dify_index, "")
+
+    await dify_index.clear()
+    monkeypatch.setattr(tree, "list_all_documents", collision_documents)
+    with pytest.raises(ValueError, match="Path collision"):
+        await tree.ensure_tree(dify_accessor, dify_index, "")
+
+
+def test_tree_slug_and_timestamp_helpers():
+    assert tree.extract_slug({
+        "doc_metadata": {
+            "slug": "a/b"
+        },
+        "name": "fallback"
+    }) == ("a/b", True)
+    assert tree.extract_slug(
+        {
+            "doc_metadata": [{
+                "name": "path",
+                "value": "docs/start"
+            }],
+            "name": "fallback"
+        }, "path") == ("docs/start", True)
+    assert tree.extract_slug(
+        {
+            "doc_metadata": {
+                "path": "docs/map"
+            },
+            "name": "fallback"
+        }, "path") == ("docs/map", True)
+    assert tree.normalize_slug("/a//b/") == "/a/b"
+    assert tree.extract_document_size(
+        {"data_source_info": {
+            "upload_file": {
+                "size": 7
+            }
+        }}) == 7
+    assert tree.timestamp_to_iso(None) == ""
+    assert tree.virtual_path("/a", "/knowledge/") == "/knowledge/a"
+    assert tree.parent("/a/b") == "/a"
+    assert tree.gnu_basename("/a/b") == "b"
+
+
+def test_normalize_slug_rejects_invalid_segments():
+    with pytest.raises(ValueError, match="Invalid empty"):
+        tree.normalize_slug("/")
+    with pytest.raises(ValueError, match="Invalid Dify document slug segment"):
+        tree.normalize_slug("../x")

python/tests/core/dify/test_walk.py

@@ -0,0 +1,44 @@
+import pytest
+
+from mirage.core.dify import tree, walk
+
+from .conftest import list_nested_documents, no_documents
+
+
+@pytest.mark.asyncio
+async def test_walk_returns_recursive_paths_with_depth_controls(
+        monkeypatch, dify_accessor, dify_index, knowledge_root):
+    monkeypatch.setattr(tree, "list_all_documents", list_nested_documents)
+
+    full = await walk.walk(dify_accessor,
+                           knowledge_root,
+                           dify_index,
+                           include_root=True,
+                           strip_prefix=True)
+    shallow = await walk.walk(dify_accessor,
+                              knowledge_root,
+                              dify_index,
+                              include_root=True,
+                              maxdepth=1,
+                              strip_prefix=True)
+
+    assert full == [
+        "/",
+        "/README.md",
+        "/guides",
+        "/guides/deep",
+        "/guides/deep/note",
+        "/guides/quickstart",
+    ]
+    assert shallow == ["/", "/README.md", "/guides"]
+
+
+@pytest.mark.asyncio
+async def test_walk_can_ignore_missing_paths(dify_accessor, dify_index,
+                                             guide_path, monkeypatch):
+    monkeypatch.setattr(tree, "list_all_documents", no_documents)
+
+    assert await walk.walk(dify_accessor,
+                           guide_path,
+                           dify_index,
+                           ignore_missing=True) == []