chore: import upstream snapshot with attribution

python/mirage/core/dify/__init__.py

@@ -0,0 +1 @@
+__all__ = []

python/mirage/core/dify/_client.py

@@ -0,0 +1,125 @@
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from typing import Any
+
+import httpx
+
+from mirage.resource.dify.config import DifyConfig
+
+logger = logging.getLogger(__name__)
+
+
+async def dify_get(config: DifyConfig,
+                   endpoint: str,
+                   params: dict[str, Any] | None = None) -> dict[str, Any]:
+    url = f"{config.base_url}{endpoint}"
+    headers = {"Authorization": f"Bearer {config.api_key}"}
+    last_error: httpx.HTTPStatusError | None = None
+    for attempt in range(4):
+        async with httpx.AsyncClient(timeout=30) as client:
+            response = await client.get(url, headers=headers, params=params)
+        if response.status_code == 429 and attempt < 3:
+            logger.warning("Dify rate limited request to %s", endpoint)
+            await asyncio.sleep(2**attempt)
+            continue
+        if 500 <= response.status_code < 600 and attempt < 1:
+            await asyncio.sleep(1)
+            continue
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            last_error = exc
+            break
+        return response.json()
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"Dify request failed: {endpoint}")
+
+
+async def dify_post(config: DifyConfig, endpoint: str,
+                    body: dict[str, Any]) -> dict[str, Any]:
+    url = f"{config.base_url}{endpoint}"
+    headers = {"Authorization": f"Bearer {config.api_key}"}
+    last_error: httpx.HTTPStatusError | None = None
+    for attempt in range(4):
+        async with httpx.AsyncClient(timeout=30) as client:
+            response = await client.post(url, headers=headers, json=body)
+        if response.status_code == 429 and attempt < 3:
+            logger.warning("Dify rate limited request to %s", endpoint)
+            await asyncio.sleep(2**attempt)
+            continue
+        if 500 <= response.status_code < 600 and attempt < 1:
+            await asyncio.sleep(1)
+            continue
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            last_error = exc
+            break
+        return response.json()
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"Dify request failed: {endpoint}")
+
+
+async def list_all_documents(config: DifyConfig) -> list[dict[str, Any]]:
+    documents: list[dict[str, Any]] = []
+    page = 1
+    while True:
+        payload = await dify_get(
+            config,
+            f"/datasets/{config.dataset_id}/documents",
+            {
+                "page": page,
+                "limit": 100
+            },
+        )
+        for document in payload.get("data") or []:
+            if is_visible_document(document):
+                documents.append(document)
+        if not payload.get("has_more"):
+            return documents
+        page += 1
+
+
+async def get_document_detail(config: DifyConfig,
+                              document_id: str) -> dict[str, Any]:
+    return await dify_get(
+        config, f"/datasets/{config.dataset_id}/documents/{document_id}")
+
+
+async def get_document_segments(config: DifyConfig,
+                                document_id: str) -> list[dict[str, Any]]:
+    segments: list[dict[str, Any]] = []
+    async for page in iter_segment_pages(config, document_id):
+        segments.extend(page)
+    return segments
+
+
+async def iter_segment_pages(
+    config: DifyConfig,
+    document_id: str,
+) -> AsyncIterator[list[dict[str, Any]]]:
+    page = 1
+    while True:
+        payload = await dify_get(
+            config,
+            f"/datasets/{config.dataset_id}/documents/{document_id}/segments",
+            {
+                "page": page,
+                "limit": 100,
+                "status": "completed",
+                "enabled": "true",
+            },
+        )
+        yield payload.get("data") or []
+        if not payload.get("has_more"):
+            return
+        page += 1
+
+
+def is_visible_document(document: dict[str, Any]) -> bool:
+    return (document.get("enabled") is True
+            and document.get("indexing_status") == "completed"
+            and document.get("archived") is False)

python/mirage/core/dify/find.py

@@ -0,0 +1,113 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
+                                               keep, start_basename,
+                                               tree_has_type)
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.stat import stat
+from mirage.core.dify.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+
+async def find(
+    accessor,
+    path: PathSpec,
+    name: str | None = None,
+    type: str | None = None,
+    min_size: int | None = None,
+    max_size: int | None = None,
+    maxdepth: int | None = None,
+    name_exclude: str | None = None,
+    or_names: list[str] | None = None,
+    mtime_min: float | None = None,
+    mtime_max: float | None = None,
+    iname: str | None = None,
+    path_pattern: str | None = None,
+    mindepth: int | None = None,
+    empty: bool = False,
+    tree: PredNode | None = None,
+    index: IndexCacheStore | None = None,
+) -> list[str]:
+    if index is None:
+        raise ValueError("find: missing index")
+    results = await walk(accessor,
+                         path,
+                         index,
+                         include_root=True,
+                         maxdepth=maxdepth,
+                         strip_prefix=True)
+    tree = tree if tree is not None else build_tree(name=name,
+                                                    iname=iname,
+                                                    path_pattern=path_pattern,
+                                                    type=type,
+                                                    name_exclude=name_exclude,
+                                                    or_names=or_names)
+    needs_kind = (tree_has_type(tree) or min_size is not None
+                  or max_size is not None)
+    start_name = start_basename(path)
+    filtered: list[str] = []
+    for item in results:
+        if await _matches(accessor, item,
+                          mount_prefix_of(path.virtual, path.resource_path),
+                          index, path.mount_path, tree, needs_kind, min_size,
+                          max_size, mindepth, start_name):
+            filtered.append(item)
+    return sorted(filtered)
+
+
+async def _matches(
+    accessor,
+    item: str,
+    prefix: str,
+    index: IndexCacheStore,
+    root: str,
+    tree: PredNode,
+    needs_kind: bool,
+    min_size: int | None,
+    max_size: int | None,
+    mindepth: int | None,
+    start_name: str,
+) -> bool:
+    root_norm = root.rstrip("/") or "/"
+    item_norm = item.rstrip("/") or "/"
+    item_name = (start_name if item_norm == root_norm else
+                 item.rstrip("/").rsplit("/", 1)[-1])
+    spec = PathSpec.from_str_path(item, mount_key(item, prefix))
+    kind = "f"
+    if needs_kind:
+        resolved = await resolve_path(accessor, spec, index)
+        kind = "d" if resolved.is_dir else "f"
+    entry = FindEntry(key=item,
+                      name=item_name,
+                      kind=kind,
+                      depth=_relative_depth(item, root))
+    if not keep(entry, tree, mindepth):
+        return False
+    # Directories count as size 0 for -size (deliberate GNU divergence).
+    if min_size is not None or max_size is not None:
+        if kind == "d":
+            size = 0
+        else:
+            item_stat = await stat(accessor, spec, index)
+            if item_stat.size is None:
+                return False
+            size = item_stat.size
+        if min_size is not None and size < min_size:
+            return False
+        if max_size is not None and size > max_size:
+            return False
+    return True
+
+
+def _relative_depth(item: str, root: str) -> int:
+    root_norm = root.rstrip("/") or "/"
+    item_norm = item.rstrip("/") or "/"
+    if item_norm == root_norm:
+        return 0
+    if root_norm == "/":
+        relative = item_norm.strip("/")
+    else:
+        relative = item_norm.removeprefix(root_norm).lstrip("/")
+    if not relative:
+        return 0
+    return relative.count("/") + 1

python/mirage/core/dify/glob.py

@@ -0,0 +1,15 @@
+from mirage.accessor.dify import DifyAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.constants import SCOPE_ERROR
+from mirage.core.dify.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: DifyAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/dify/grep.py

@@ -0,0 +1,40 @@
+import re
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify.read import read_stream
+from mirage.io.async_line_iterator import AsyncLineIterator
+from mirage.types import PathSpec
+
+
+async def grep_bytes(
+        accessor,
+        paths: list[PathSpec],
+        pattern: str,
+        index: IndexCacheStore,
+        ignore_case: bool = False) -> tuple[bytes, dict[str, bytes]]:
+    flags = re.IGNORECASE if ignore_case else 0
+    regex = re.compile(pattern, flags)
+    lines: list[str] = []
+    reads: dict[str, bytes] = {}
+    for path in paths:
+        chunks: list[bytes] = []
+        stream = _record_chunks(read_stream(accessor, path, index), chunks)
+        async for line_number, raw_line in _enumerate_lines(stream):
+            line = raw_line.decode(errors="replace")
+            if regex.search(line):
+                lines.append(f"{path.virtual}:{line_number}:{line}")
+        reads[path.virtual] = b"".join(chunks)
+    return "\n".join(lines).encode(), reads
+
+
+async def _record_chunks(source, chunks: list[bytes]):
+    async for chunk in source:
+        chunks.append(chunk)
+        yield chunk
+
+
+async def _enumerate_lines(source):
+    line_number = 0
+    async for raw_line in AsyncLineIterator(source):
+        line_number += 1
+        yield line_number, raw_line

python/mirage/core/dify/path.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.dify.tree import ensure_tree
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+@dataclass(frozen=True)
+class ResolvedDifyPath:
+    virtual_key: str
+    mount_prefix: str
+    is_dir: bool
+    entry: IndexEntry | None = None
+
+
+async def resolve_path(accessor, path: PathSpec,
+                       index: IndexCacheStore) -> ResolvedDifyPath:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    mount_prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+    await ensure_tree(accessor, index, mount_prefix)
+    virtual_key = virtual_key_for(path)
+    result = await index.get(virtual_key)
+    if result.entry is not None:
+        return ResolvedDifyPath(
+            virtual_key=virtual_key,
+            mount_prefix=mount_prefix,
+            is_dir=result.entry.resource_type == "folder",
+            entry=result.entry,
+        )
+    listing = await index.list_dir(virtual_key)
+    if listing.entries is not None:
+        return ResolvedDifyPath(virtual_key=virtual_key,
+                                mount_prefix=mount_prefix,
+                                is_dir=True)
+    raise enoent(path)
+
+
+def virtual_key_for(path: PathSpec) -> str:
+    raw = path.directory if path.pattern else path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+    if prefix:
+        root = prefix.rstrip("/") or "/"
+        if raw == root or raw.startswith(root + "/"):
+            return raw.rstrip("/") or root
+        rest = raw.strip("/")
+        if not rest:
+            return root
+        return root + "/" + rest
+    stripped = raw.strip("/")
+    return "/" + stripped if stripped else "/"

python/mirage/core/dify/read.py

@@ -0,0 +1,45 @@
+import errno
+from collections.abc import AsyncIterator
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify._client import get_document_segments, iter_segment_pages
+from mirage.core.dify.path import resolve_path
+from mirage.types import PathSpec
+
+
+async def read_bytes(accessor, path: PathSpec,
+                     index: IndexCacheStore) -> bytes:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
+    segments = await get_document_segments(accessor.config, resolved.entry.id)
+    return segments_to_bytes(segments)
+
+
+async def read_stream(accessor, path: PathSpec,
+                      index: IndexCacheStore) -> AsyncIterator[bytes]:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
+    first = True
+    async for page in iter_segment_pages(accessor.config, resolved.entry.id):
+        for segment in page:
+            if first:
+                first = False
+            else:
+                yield b"\n"
+            yield segment_text(segment).encode()
+
+
+def segments_to_bytes(segments: list[dict[str, Any]]) -> bytes:
+    return "\n".join(segment_text(segment) for segment in segments).encode()
+
+
+def segment_text(segment: dict[str, Any]) -> str:
+    value = segment.get("content")
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return str(value)

python/mirage/core/dify/readdir.py

@@ -0,0 +1,15 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify.path import resolve_path
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent, enotdir
+
+
+async def readdir(accessor, path: PathSpec,
+                  index: IndexCacheStore) -> list[str]:
+    resolved = await resolve_path(accessor, path, index)
+    if not resolved.is_dir:
+        raise enotdir(path)
+    listing = await index.list_dir(resolved.virtual_key)
+    if listing.entries is None:
+        raise enoent(path)
+    return listing.entries

python/mirage/core/dify/search.py

@@ -0,0 +1,224 @@
+import logging
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.dify._client import dify_post
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.read import segment_text
+from mirage.core.dify.tree import normalize_slug
+from mirage.core.dify.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_prefix_of, rekey
+from mirage.utils.score import format_score
+
+logger = logging.getLogger(__name__)
+
+METHODS = {
+    "semantic": "semantic_search",
+    "fulltext": "full_text_search",
+    "hybrid": "hybrid_search",
+    "keyword": "keyword_search",
+}
+
+
+async def search_segments(
+    accessor,
+    query: str,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+    method: str = "semantic",
+    top_k: int = 10,
+    threshold: float = 0.0,
+    mount_prefix: str = "",
+) -> bytes:
+    search_method = validate_args(query, method, top_k, threshold)
+    if not mount_prefix and paths:
+        mount_prefix = mount_prefix_of(paths[0].virtual,
+                                       paths[0].resource_path)
+    retrieval_model = {
+        "search_method": search_method,
+        "top_k": min(top_k, 100),
+        "score_threshold_enabled": threshold > 0,
+        "score_threshold": threshold,
+        "reranking_enable": False,
+    }
+    has_name_based_target = False
+    if paths:
+        conditions, has_name_based_target = await metadata_conditions(
+            accessor, paths, index)
+        if not conditions:
+            return b""
+        retrieval_model["metadata_filtering_conditions"] = {
+            "logical_operator": "or",
+            "conditions": conditions,
+        }
+    response = await dify_post(
+        accessor.config,
+        f"/datasets/{accessor.config.dataset_id}/retrieve",
+        {
+            "query": query,
+            "retrieval_model": retrieval_model
+        },
+    )
+    output = records_to_bytes(
+        response.get("records") or [], accessor.config.slug_metadata_name,
+        mount_prefix)
+    if paths and has_name_based_target and output == b"":
+        logger.debug(
+            "Dify scoped search returned no records for name-based documents; "
+            "check that Built-in Fields are enabled in Dify dataset metadata.")
+    return output
+
+
+def validate_args(query: str, method: str, top_k: int,
+                  threshold: float) -> str:
+    if not query:
+        raise ValueError("search: query is required")
+    if len(query) > 250:
+        raise ValueError("search: query cannot exceed 250 characters")
+    if top_k <= 0:
+        raise ValueError("search: top-k must be positive")
+    if threshold < 0 or threshold > 1:
+        raise ValueError("search: threshold must be in [0, 1]")
+    if method not in METHODS:
+        raise ValueError(
+            "search: method must be one of semantic, fulltext, hybrid, keyword"
+        )
+    return METHODS[method]
+
+
+async def metadata_conditions(
+    accessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> tuple[list[dict], bool]:
+    targets = await target_entries(accessor, paths, index)
+    slug_values: list[str] = []
+    name_values: list[str] = []
+    for entry in targets.values():
+        if entry.extra.get("has_slug") is True:
+            slug_values.append(str(entry.extra["raw_slug"]))
+        else:
+            name_values.append(entry.name)
+    conditions: list[dict] = []
+    if slug_values:
+        conditions.append({
+            "name": accessor.config.slug_metadata_name,
+            "comparison_operator": "in",
+            "value": sorted(slug_values),
+        })
+    if name_values:
+        conditions.append({
+            "name": "document_name",
+            "comparison_operator": "in",
+            "value": sorted(name_values),
+        })
+    return conditions, bool(name_values)
+
+
+async def target_entries(
+    accessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> dict[str, IndexEntry]:
+    targets: dict[str, IndexEntry] = {}
+    for path in paths:
+        resolved = await resolve_path(accessor, path, index)
+        if resolved.entry is not None and not resolved.is_dir:
+            targets[resolved.entry.id] = resolved.entry
+            continue
+        if resolved.is_dir:
+            children = await walk(accessor,
+                                  path,
+                                  index,
+                                  include_root=False,
+                                  strip_prefix=False)
+            for child in children:
+                child_spec = PathSpec.from_str_path(
+                    child, rekey(path.virtual, path.resource_path, child))
+                child_resolved = await resolve_path(accessor, child_spec,
+                                                    index)
+                if (child_resolved.entry is not None
+                        and not child_resolved.is_dir):
+                    targets[child_resolved.entry.id] = child_resolved.entry
+    return targets
+
+
+def records_to_bytes(
+    records: list[dict[str, Any]],
+    slug_metadata_name: str,
+    mount_prefix: str,
+) -> bytes:
+    contents: list[str] = []
+    for record in records:
+        segment = record.get("segment")
+        if not isinstance(segment, dict):
+            continue
+        header = format_record_header(record, slug_metadata_name, mount_prefix)
+        if header is None:
+            continue
+        content = segment_text(segment)
+        contents.append(f"{header}\n{content}")
+    if not contents:
+        return b""
+    return ("\n".join(contents) + "\n").encode()
+
+
+def format_record_header(
+    record: dict[str, Any],
+    slug_metadata_name: str,
+    mount_prefix: str,
+) -> str | None:
+    path = record_path(record, slug_metadata_name, mount_prefix)
+    if path is None:
+        return None
+    score = format_score(record.get("score"))
+    if score is None:
+        return path
+    return f"{path}:{score}"
+
+
+def record_path(
+    record: dict[str, Any],
+    slug_metadata_name: str,
+    mount_prefix: str,
+) -> str | None:
+    segment = record.get("segment")
+    if not isinstance(segment, dict):
+        return None
+    document = segment.get("document")
+    if not isinstance(document, dict):
+        return None
+    raw_path = document_path(document, slug_metadata_name)
+    if raw_path is None:
+        return None
+    try:
+        normalized = normalize_slug(raw_path)
+    except ValueError:
+        logger.debug("Skipping Dify record with invalid slug/name: %r",
+                     raw_path)
+        return None
+    prefix = mount_prefix.rstrip("/")
+    if not prefix:
+        return normalized
+    return prefix + normalized
+
+
+def document_path(
+    document: dict[str, Any],
+    slug_metadata_name: str,
+) -> str | None:
+    metadata = document.get("doc_metadata")
+    if isinstance(metadata, list):
+        for item in metadata:
+            if (isinstance(item, dict)
+                    and item.get("name") == slug_metadata_name
+                    and item.get("value") is not None):
+                return str(item["value"])
+    if isinstance(metadata,
+                  dict) and metadata.get(slug_metadata_name) is not None:
+        return str(metadata[slug_metadata_name])
+    name = document.get("name")
+    if name is None:
+        return None
+    return str(name)

python/mirage/core/dify/stat.py

@@ -0,0 +1,77 @@
+from datetime import datetime, timezone
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify._client import get_document_detail
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.tree import extract_document_size
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+
+
+async def stat_light(accessor, path: PathSpec,
+                     index: IndexCacheStore) -> FileStat:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        return FileStat(
+            name=stat_name(resolved.virtual_key, resolved.mount_prefix),
+            type=FileType.DIRECTORY,
+            extra={"children_count": 0},
+        )
+    if resolved.entry is None:
+        raise enoent(path)
+    return FileStat(
+        name=resolved.entry.name,
+        type=FileType.TEXT,
+        size=resolved.entry.size,
+        modified=timestamp_to_zulu(resolved.entry.remote_time),
+        fingerprint=None,
+        revision=None,
+        extra=dict(resolved.entry.extra),
+    )
+
+
+async def stat(accessor, path: PathSpec, index: IndexCacheStore) -> FileStat:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        return FileStat(
+            name=stat_name(resolved.virtual_key, resolved.mount_prefix),
+            type=FileType.DIRECTORY,
+            extra={"children_count": 0},
+        )
+    if resolved.entry is None:
+        raise enoent(path)
+    detail = await get_document_detail(accessor.config, resolved.entry.id)
+    size = extract_document_size(detail)
+    if size is None:
+        size = resolved.entry.size
+    extra = dict(resolved.entry.extra)
+    extra["document_id"] = resolved.entry.id
+    if "tokens" in detail:
+        extra["tokens"] = detail.get("tokens")
+    if "indexing_status" in detail:
+        extra["indexing_status"] = detail.get("indexing_status")
+    return FileStat(
+        name=resolved.entry.name,
+        type=FileType.TEXT,
+        size=size,
+        modified=timestamp_to_zulu(detail.get("updated_at")),
+        fingerprint=None,
+        revision=None,
+        extra=extra,
+    )
+
+
+def timestamp_to_zulu(value: object) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        return datetime.fromtimestamp(
+            value, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    return str(value)
+
+
+def stat_name(virtual_key: str, mount_prefix: str) -> str:
+    root = mount_prefix.rstrip("/") or "/"
+    if virtual_key == root:
+        return "/"
+    return virtual_key.rstrip("/").rsplit("/", 1)[-1]

python/mirage/core/dify/tree.py

@@ -0,0 +1,174 @@
+from datetime import datetime, timezone
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.dify._client import list_all_documents
+from mirage.utils.path import gnu_basename, parent
+
+
+async def ensure_tree(accessor,
+                      index: IndexCacheStore,
+                      prefix: str = "") -> None:
+    root_key = mount_root(prefix)
+    listing = await index.list_dir(root_key)
+    if listing.entries is not None:
+        return
+
+    # list_all_documents already filters to visible documents.
+    documents = await list_all_documents(accessor.config)
+    dir_entries = build_dir_entries(
+        documents,
+        prefix,
+        accessor.config.slug_metadata_name,
+    )
+    for directory in sorted(dir_entries):
+        await index.set_dir(
+            directory, sorted(dir_entries[directory],
+                              key=lambda item: item[0]))
+
+
+def build_dir_entries(
+    documents: list[dict[str, Any]],
+    prefix: str,
+    slug_metadata_name: str = "slug",
+) -> dict[str, list[tuple[str, IndexEntry]]]:
+    files: dict[str, dict[str, Any]] = {}
+    raw_slugs: dict[str, str] = {}
+    has_slugs: dict[str, bool] = {}
+    for document in documents:
+        slug, has_slug = extract_slug(document, slug_metadata_name)
+        path = normalize_slug(slug)
+        if path in files:
+            value = path.strip("/")
+            raise ValueError(
+                f"Duplicate {slug_metadata_name} '{value}': documents "
+                f"'{files[path].get('id')}' and '{document.get('id')}' share "
+                "the same path.")
+        files[path] = document
+        raw_slugs[path] = str(slug)
+        has_slugs[path] = has_slug
+
+    raise_on_collisions(files)
+    directories = collect_directories(set(files))
+    dir_entries: dict[str, list[tuple[str, IndexEntry]]] = {
+        virtual_path(directory, prefix): []
+        for directory in directories
+    }
+    for directory in sorted(directories):
+        if directory == "/":
+            continue
+        entry = IndexEntry(
+            id=directory.strip("/"),
+            name=gnu_basename(directory),
+            resource_type="folder",
+        )
+        dir_entries[virtual_path(parent(directory), prefix)].append(
+            (entry.name, entry))
+
+    for path, document in sorted(files.items()):
+        entry = IndexEntry(
+            id=str(document["id"]),
+            name=gnu_basename(path),
+            resource_type="file",
+            size=extract_document_size(document),
+            remote_time=timestamp_to_iso(document.get("created_at")),
+            extra={
+                "slug": path.strip("/"),
+                "slug_metadata_name": slug_metadata_name,
+                "raw_slug": raw_slugs[path],
+                "has_slug": has_slugs[path],
+                "tokens": document.get("tokens"),
+                "indexing_status": document.get("indexing_status"),
+                "data_source_type": document.get("data_source_type"),
+            },
+        )
+        dir_entries[virtual_path(parent(path), prefix)].append(
+            (entry.name, entry))
+    return dir_entries
+
+
+def extract_slug(document: dict[str, Any],
+                 slug_metadata_name: str = "slug") -> tuple[str, bool]:
+    metadata = document.get("doc_metadata")
+    if isinstance(metadata, list):
+        for item in metadata:
+            if (isinstance(item, dict)
+                    and item.get("name") == slug_metadata_name):
+                value = item.get("value")
+                if value is not None:
+                    return str(value), True
+    if (isinstance(metadata, dict)
+            and metadata.get(slug_metadata_name) is not None):
+        return str(metadata[slug_metadata_name]), True
+    return str(document["name"]), False
+
+
+def normalize_slug(value: str) -> str:
+    parts = [part for part in value.strip("/").split("/") if part]
+    if not parts:
+        raise ValueError("Invalid empty Dify document slug.")
+    invalid = {".", ".."}
+    for part in parts:
+        if part in invalid:
+            raise ValueError(f"Invalid Dify document slug segment: {part!r}")
+    return "/" + "/".join(parts)
+
+
+def raise_on_collisions(files: dict[str, dict[str, Any]]) -> None:
+    paths = set(files)
+    for path in sorted(paths):
+        parts = path.strip("/").split("/")
+        for index in range(1, len(parts)):
+            ancestor = "/" + "/".join(parts[:index])
+            if ancestor in paths:
+                raise ValueError(
+                    "Path collision: document "
+                    f"'{files[ancestor].get('id')}' uses file path "
+                    f"'{ancestor.strip('/')}' but document "
+                    f"'{files[path].get('id')}' requires it as a directory "
+                    "prefix.")
+
+
+def collect_directories(paths: set[str]) -> set[str]:
+    directories = {"/"}
+    for path in paths:
+        parts = path.strip("/").split("/")
+        for index in range(1, len(parts)):
+            directories.add("/" + "/".join(parts[:index]))
+    return directories
+
+
+def extract_document_size(document: dict[str, Any]) -> int | None:
+    candidates = (
+        document.get("data_source_detail_dict"),
+        document.get("data_source_info"),
+    )
+    for candidate in candidates:
+        if isinstance(candidate, dict):
+            upload_file = candidate.get("upload_file")
+            if isinstance(upload_file, dict):
+                size = upload_file.get("size")
+                if isinstance(size, int):
+                    return size
+    return None
+
+
+def timestamp_to_iso(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, (int, float)):
+        return datetime.fromtimestamp(value, timezone.utc).isoformat()
+    return str(value)
+
+
+def mount_root(prefix: str) -> str:
+    return prefix.rstrip("/") or "/"
+
+
+def virtual_path(path: str, prefix: str) -> str:
+    root = mount_root(prefix)
+    if path == "/":
+        return root
+    if root == "/":
+        return path
+    return root + path

python/mirage/core/dify/walk.py

@@ -0,0 +1,49 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import rekey
+
+
+async def walk(
+    accessor,
+    path: PathSpec,
+    index: IndexCacheStore,
+    *,
+    include_root: bool = False,
+    maxdepth: int | None = None,
+    strip_prefix: bool = False,
+    ignore_missing: bool = False,
+    depth: int = 0,
+) -> list[str]:
+    try:
+        resolved = await resolve_path(accessor, path, index)
+    except (FileNotFoundError, NotADirectoryError):
+        if ignore_missing:
+            return []
+        raise
+
+    current = path.mount_path if strip_prefix else path.virtual
+    results = [current] if include_root else []
+    if not resolved.is_dir or (maxdepth is not None and depth >= maxdepth):
+        return results
+
+    try:
+        children = await readdir(accessor, path, index)
+    except (FileNotFoundError, NotADirectoryError):
+        if ignore_missing:
+            return results
+        raise
+
+    for child in children:
+        child_path = PathSpec.from_str_path(
+            child, rekey(path.virtual, path.resource_path, child))
+        results.extend(await walk(accessor,
+                                  child_path,
+                                  index,
+                                  include_root=True,
+                                  maxdepth=maxdepth,
+                                  strip_prefix=strip_prefix,
+                                  ignore_missing=ignore_missing,
+                                  depth=depth + 1))
+    return results