chore: import upstream snapshot with attribution

python/mirage/core/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/mirage/core/chroma/__init__.py

@@ -0,0 +1 @@
+__all__ = []

python/mirage/core/chroma/_client.py

@@ -0,0 +1,95 @@
+from collections.abc import AsyncIterator
+from typing import Any
+
+PATH_TREE_ID = "__path_tree__"
+PAGE_CHUNK_BATCH_SIZE = 100
+
+
+async def fetch_path_tree(accessor) -> str:
+    collection = await accessor.get_collection()
+    result = await collection.get(ids=[PATH_TREE_ID])
+    documents = result.get("documents") or []
+    if not documents:
+        raise FileNotFoundError(PATH_TREE_ID)
+    value = documents[0]
+    if value is None:
+        raise FileNotFoundError(PATH_TREE_ID)
+    if isinstance(value, str):
+        return value
+    return str(value)
+
+
+async def fetch_page_chunks(accessor, slug: str) -> str:
+    chunks = await page_chunks(accessor, slug)
+    return "\n".join(chunk["document"] for chunk in chunks)
+
+
+async def iter_page_chunks(accessor, slug: str) -> AsyncIterator[str]:
+    chunks = await page_chunks(accessor, slug)
+    for chunk in chunks:
+        yield chunk["document"]
+
+
+async def page_chunks(accessor, slug: str) -> list[dict[str, Any]]:
+    collection = await accessor.get_collection()
+    chunks: list[dict[str, Any]] = []
+    offset = 0
+    while True:
+        result = await collection.get(
+            where={accessor.config.slug_field: slug},
+            include=["documents", "metadatas"],
+            limit=PAGE_CHUNK_BATCH_SIZE,
+            offset=offset,
+        )
+        documents = result.get("documents") or []
+        metadatas = result.get("metadatas") or [{} for _ in documents]
+        for document, metadata in zip(documents, metadatas, strict=True):
+            chunks.append({
+                "document": "" if document is None else str(document),
+                "metadata": metadata if isinstance(metadata, dict) else {},
+            })
+        if len(documents) < PAGE_CHUNK_BATCH_SIZE:
+            break
+        offset += PAGE_CHUNK_BATCH_SIZE
+    return sorted(chunks,
+                  key=lambda item: chunk_index(
+                      item["metadata"], accessor.config.chunk_index_field))
+
+
+async def query_contains(
+    accessor,
+    pattern: str,
+    candidate_slugs: list[str],
+    *,
+    regex: bool = False,
+) -> list[str]:
+    if not candidate_slugs:
+        return []
+    collection = await accessor.get_collection()
+    result = await collection.get(
+        where={accessor.config.slug_field: {
+            "$in": candidate_slugs
+        }},
+        where_document={"$regex" if regex else "$contains": pattern},
+        include=["metadatas"],
+    )
+    matched: set[str] = set()
+    for metadata in result.get("metadatas") or []:
+        if isinstance(metadata, dict):
+            slug = metadata.get(accessor.config.slug_field)
+            if slug is not None:
+                matched.add(str(slug))
+    return sorted(matched)
+
+
+def chunk_index(metadata: dict[str, Any], field: str) -> int:
+    value = metadata.get(field, 0)
+    if isinstance(value, bool):
+        return 0
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(value)
+    if isinstance(value, str) and value.isdigit():
+        return int(value)
+    return 0

python/mirage/core/chroma/find.py

@@ -0,0 +1,113 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
+                                               keep, start_basename,
+                                               tree_has_type)
+from mirage.core.chroma.path import resolve_path
+from mirage.core.chroma.stat import stat
+from mirage.core.chroma.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+
+async def find(
+    accessor,
+    path: PathSpec,
+    name: str | None = None,
+    type: str | None = None,
+    min_size: int | None = None,
+    max_size: int | None = None,
+    maxdepth: int | None = None,
+    name_exclude: str | None = None,
+    or_names: list[str] | None = None,
+    mtime_min: float | None = None,
+    mtime_max: float | None = None,
+    iname: str | None = None,
+    path_pattern: str | None = None,
+    mindepth: int | None = None,
+    empty: bool = False,
+    tree: PredNode | None = None,
+    index: IndexCacheStore | None = None,
+) -> list[str]:
+    if index is None:
+        raise ValueError("find: missing index")
+    results = await walk(accessor,
+                         path,
+                         index,
+                         include_root=True,
+                         maxdepth=maxdepth,
+                         strip_prefix=True)
+    tree = tree if tree is not None else build_tree(name=name,
+                                                    iname=iname,
+                                                    path_pattern=path_pattern,
+                                                    type=type,
+                                                    name_exclude=name_exclude,
+                                                    or_names=or_names)
+    needs_kind = (tree_has_type(tree) or min_size is not None
+                  or max_size is not None)
+    start_name = start_basename(path)
+    filtered: list[str] = []
+    for item in results:
+        if await _matches(accessor, item,
+                          mount_prefix_of(path.virtual, path.resource_path),
+                          index, path.mount_path, tree, needs_kind, min_size,
+                          max_size, mindepth, start_name):
+            filtered.append(item)
+    return sorted(filtered)
+
+
+async def _matches(
+    accessor,
+    item: str,
+    prefix: str,
+    index: IndexCacheStore,
+    root: str,
+    tree: PredNode,
+    needs_kind: bool,
+    min_size: int | None,
+    max_size: int | None,
+    mindepth: int | None,
+    start_name: str,
+) -> bool:
+    root_norm = root.rstrip("/") or "/"
+    item_norm = item.rstrip("/") or "/"
+    item_name = (start_name if item_norm == root_norm else
+                 item.rstrip("/").rsplit("/", 1)[-1])
+    spec = PathSpec.from_str_path(item, mount_key(item, prefix))
+    kind = "f"
+    if needs_kind:
+        resolved = await resolve_path(accessor, spec, index)
+        kind = "d" if resolved.is_dir else "f"
+    entry = FindEntry(key=item,
+                      name=item_name,
+                      kind=kind,
+                      depth=_relative_depth(item, root))
+    if not keep(entry, tree, mindepth):
+        return False
+    # Directories count as size 0 for -size (deliberate GNU divergence).
+    if min_size is not None or max_size is not None:
+        if kind == "d":
+            size = 0
+        else:
+            item_stat = await stat(accessor, spec, index)
+            if item_stat.size is None:
+                return False
+            size = item_stat.size
+        if min_size is not None and size < min_size:
+            return False
+        if max_size is not None and size > max_size:
+            return False
+    return True
+
+
+def _relative_depth(item: str, root: str) -> int:
+    root_norm = root.rstrip("/") or "/"
+    item_norm = item.rstrip("/") or "/"
+    if item_norm == root_norm:
+        return 0
+    if root_norm == "/":
+        relative = item_norm.strip("/")
+    else:
+        relative = item_norm.removeprefix(root_norm).lstrip("/")
+    if not relative:
+        return 0
+    return relative.count("/") + 1

python/mirage/core/chroma/glob.py

@@ -0,0 +1,15 @@
+from mirage.accessor.chroma import ChromaAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.constants import SCOPE_ERROR
+from mirage.core.chroma.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: ChromaAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/chroma/grep.py

@@ -0,0 +1,122 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.grep_helper import compile_pattern, grep_lines
+from mirage.commands.builtin.utils.lines import split_lines
+from mirage.core.chroma._client import fetch_page_chunks, query_contains
+from mirage.core.chroma.path import resolve_path
+from mirage.core.chroma.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key, mount_prefix_of, rekey
+
+
+async def grep_bytes(
+    accessor,
+    paths: list[PathSpec],
+    pattern: str,
+    index: IndexCacheStore,
+    ignore_case: bool = False,
+    invert: bool = False,
+    line_numbers: bool = True,
+    count_only: bool = False,
+    files_only: bool = False,
+    whole_word: bool = False,
+    fixed_string: bool = False,
+    only_matching: bool = False,
+    max_count: int | None = None,
+) -> tuple[bytes, dict[str, bytes]]:
+    """Filename-prefixed grep used by the FUSE ops layer.
+
+    The grep command does not use this; it delegates to the generic
+    grep with the same pushdown (see commands/builtin/chroma/grep.py).
+
+    Args:
+        accessor: Chroma accessor.
+        paths (list[PathSpec]): Files or directories to search.
+        pattern (str): Pattern text.
+        index (IndexCacheStore): Cache index for path resolution.
+        ignore_case (bool): `-i`, case-insensitive matching.
+        invert (bool): `-v`, select non-matching lines.
+        line_numbers (bool): `-n`, prefix line numbers.
+        count_only (bool): `-c`, output match counts.
+        files_only (bool): `-l`, output only matching file paths.
+        whole_word (bool): `-w`, match whole words.
+        fixed_string (bool): `-F`, treat pattern as a literal string.
+        only_matching (bool): `-o`, output only matched text.
+        max_count (int | None): `-m`, stop after this many matches.
+
+    Returns:
+        tuple[bytes, dict[str, bytes]]: Output and per-file reads keyed
+            by mount-relative path.
+    """
+    regex = compile_pattern(pattern, ignore_case, fixed_string, whole_word)
+    targets = await target_slugs(accessor, paths, index)
+    mount_prefix = mount_prefix_of(paths[0].virtual,
+                                   paths[0].resource_path) if paths else ""
+    lines: list[str] = []
+    reads: dict[str, bytes] = {}
+    slug_to_path = {slug: path for path, slug in targets.items()}
+    matched_slugs = await coarse_filter_slugs(accessor,
+                                              pattern,
+                                              targets,
+                                              ignore_case=ignore_case,
+                                              invert=invert,
+                                              fixed_string=fixed_string)
+    for slug in matched_slugs:
+        content = await fetch_page_chunks(accessor, slug)
+        path = slug_to_path.get(slug, "/" + slug)
+        data = content.encode()
+        reads[PathSpec.from_str_path(path, mount_key(
+            path, mount_prefix)).mount_path] = data
+        hits = grep_lines(path, split_lines(content), regex, invert,
+                          line_numbers, count_only, files_only, only_matching,
+                          max_count)
+        if count_only:
+            if hits:
+                lines.append(f"{path}:{hits[0]}")
+        elif files_only:
+            lines.extend(hits)
+        else:
+            lines.extend(f"{path}:{hit}" for hit in hits)
+    return "\n".join(lines).encode(), reads
+
+
+async def coarse_filter_slugs(
+    accessor,
+    pattern: str,
+    targets: dict[str, str],
+    *,
+    ignore_case: bool,
+    invert: bool,
+    fixed_string: bool,
+) -> list[str]:
+    candidate_slugs = sorted(targets.values())
+    if ignore_case or invert:
+        return candidate_slugs
+    return await query_contains(accessor,
+                                pattern,
+                                candidate_slugs,
+                                regex=not fixed_string)
+
+
+async def target_slugs(accessor, paths: list[PathSpec],
+                       index: IndexCacheStore) -> dict[str, str]:
+    targets: dict[str, str] = {}
+    for path in paths:
+        resolved = await resolve_path(accessor, path, index)
+        if resolved.entry is not None and not resolved.is_dir:
+            targets[path.virtual] = str(resolved.entry.extra["slug"])
+            continue
+        if resolved.is_dir:
+            children = await walk(accessor,
+                                  path,
+                                  index,
+                                  include_root=False,
+                                  strip_prefix=False)
+            for child in children:
+                child_spec = PathSpec.from_str_path(
+                    child, rekey(path.virtual, path.resource_path, child))
+                child_resolved = await resolve_path(accessor, child_spec,
+                                                    index)
+                if (child_resolved.entry is not None
+                        and not child_resolved.is_dir):
+                    targets[child] = str(child_resolved.entry.extra["slug"])
+    return targets

python/mirage/core/chroma/path.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.chroma.tree import ensure_tree
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+@dataclass(frozen=True)
+class ResolvedChromaPath:
+    virtual_key: str
+    mount_prefix: str
+    is_dir: bool
+    entry: IndexEntry | None = None
+
+
+async def resolve_path(accessor, path: PathSpec,
+                       index: IndexCacheStore) -> ResolvedChromaPath:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    mount_prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+    await ensure_tree(accessor, index, mount_prefix)
+    virtual_key = virtual_key_for(path)
+    result = await index.get(virtual_key)
+    if result.entry is not None:
+        return ResolvedChromaPath(
+            virtual_key=virtual_key,
+            mount_prefix=mount_prefix,
+            is_dir=result.entry.resource_type == "folder",
+            entry=result.entry,
+        )
+    listing = await index.list_dir(virtual_key)
+    if listing.entries is not None:
+        return ResolvedChromaPath(virtual_key=virtual_key,
+                                  mount_prefix=mount_prefix,
+                                  is_dir=True)
+    raise enoent(path)
+
+
+def virtual_key_for(path: PathSpec) -> str:
+    raw = path.directory if path.pattern else path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+    if prefix:
+        root = prefix.rstrip("/") or "/"
+        if raw == root or raw.startswith(root + "/"):
+            return raw.rstrip("/") or root
+        rest = raw.strip("/")
+        if not rest:
+            return root
+        return root + "/" + rest
+    stripped = raw.strip("/")
+    return "/" + stripped if stripped else "/"

python/mirage/core/chroma/read.py

@@ -0,0 +1,31 @@
+import errno
+from collections.abc import AsyncIterator
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.chroma._client import fetch_page_chunks, iter_page_chunks
+from mirage.core.chroma.path import resolve_path
+from mirage.types import PathSpec
+
+
+async def read_bytes(accessor, path: PathSpec,
+                     index: IndexCacheStore) -> bytes:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
+    text = await fetch_page_chunks(accessor, resolved.entry.extra["slug"])
+    return text.encode()
+
+
+async def read_stream(accessor, path: PathSpec,
+                      index: IndexCacheStore) -> AsyncIterator[bytes]:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
+    first = True
+    async for chunk in iter_page_chunks(accessor,
+                                        resolved.entry.extra["slug"]):
+        if first:
+            first = False
+        else:
+            yield b"\n"
+        yield chunk.encode()

python/mirage/core/chroma/readdir.py

@@ -0,0 +1,15 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.chroma.path import resolve_path
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent, enotdir
+
+
+async def readdir(accessor, path: PathSpec,
+                  index: IndexCacheStore) -> list[str]:
+    resolved = await resolve_path(accessor, path, index)
+    if not resolved.is_dir:
+        raise enotdir(path)
+    listing = await index.list_dir(resolved.virtual_key)
+    if listing.entries is None:
+        raise enoent(path)
+    return listing.entries

python/mirage/core/chroma/search.py

@@ -0,0 +1,121 @@
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.chroma.path import resolve_path
+from mirage.core.chroma.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_prefix_of, rekey
+from mirage.utils.score import score_from_distance
+
+
+async def search_segments(
+    accessor,
+    query: str,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+    top_k: int = 10,
+    mount_prefix: str = "",
+) -> bytes:
+    validate_args(query, top_k)
+    if not mount_prefix and paths:
+        mount_prefix = mount_prefix_of(paths[0].virtual,
+                                       paths[0].resource_path)
+    kwargs: dict[str, Any] = {
+        "query_texts": [query],
+        "n_results": top_k,
+        "include": ["documents", "metadatas", "distances"],
+    }
+    scoped_slugs: set[str] | None = None
+    if paths:
+        scoped_slugs = set((await target_entries(accessor, paths,
+                                                 index)).keys())
+        if not scoped_slugs:
+            return b""
+        kwargs["where"] = {
+            accessor.config.slug_field: {
+                "$in": sorted(scoped_slugs)
+            }
+        }
+    collection = await accessor.get_collection()
+    response = await collection.query(**kwargs)
+    return query_result_to_bytes(response, accessor.config.slug_field,
+                                 mount_prefix, scoped_slugs)
+
+
+def validate_args(query: str, top_k: int) -> None:
+    if not query:
+        raise ValueError("search: query is required")
+    if len(query) > 250:
+        raise ValueError("search: query cannot exceed 250 characters")
+    if top_k <= 0:
+        raise ValueError("search: top-k must be positive")
+
+
+async def target_entries(
+    accessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> dict[str, IndexEntry]:
+    targets: dict[str, IndexEntry] = {}
+    for path in paths:
+        resolved = await resolve_path(accessor, path, index)
+        if resolved.entry is not None and not resolved.is_dir:
+            targets[str(resolved.entry.extra["slug"])] = resolved.entry
+            continue
+        if resolved.is_dir:
+            children = await walk(accessor,
+                                  path,
+                                  index,
+                                  include_root=False,
+                                  strip_prefix=False)
+            for child in children:
+                child_spec = PathSpec.from_str_path(
+                    child, rekey(path.virtual, path.resource_path, child))
+                child_resolved = await resolve_path(accessor, child_spec,
+                                                    index)
+                if (child_resolved.entry is not None
+                        and not child_resolved.is_dir):
+                    targets[str(child_resolved.entry.extra["slug"]
+                                )] = child_resolved.entry
+    return targets
+
+
+def query_result_to_bytes(
+    response: dict[str, Any],
+    slug_field: str,
+    mount_prefix: str,
+    scoped_slugs: set[str] | None = None,
+) -> bytes:
+    documents = first_result_list(response.get("documents"))
+    metadatas = first_result_list(response.get("metadatas"))
+    distances = first_result_list(response.get("distances"))
+    contents: list[str] = []
+    for index, document in enumerate(documents):
+        metadata = metadatas[index] if index < len(metadatas) else {}
+        if not isinstance(metadata, dict):
+            continue
+        slug = metadata.get(slug_field)
+        if slug is None:
+            continue
+        slug_value = str(slug).strip("/")
+        if scoped_slugs is not None and slug_value not in scoped_slugs:
+            continue
+        score = score_from_distance(distances[index] if index <
+                                    len(distances) else None)
+        path = "/" + slug_value
+        prefix = mount_prefix.rstrip("/")
+        if prefix:
+            path = prefix + path
+        content = "" if document is None else str(document)
+        contents.append(f"{path}:{score}\n{content}")
+    if not contents:
+        return b""
+    return ("\n".join(contents) + "\n").encode()
+
+
+def first_result_list(value: object) -> list:
+    if not isinstance(value, list):
+        return []
+    if value and isinstance(value[0], list):
+        return value[0]
+    return value

python/mirage/core/chroma/stat.py

@@ -0,0 +1,37 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.chroma.path import resolve_path
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+
+
+async def stat_light(accessor, path: PathSpec,
+                     index: IndexCacheStore) -> FileStat:
+    return await stat(accessor, path, index)
+
+
+async def stat(accessor, path: PathSpec, index: IndexCacheStore) -> FileStat:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        return FileStat(
+            name=stat_name(resolved.virtual_key, resolved.mount_prefix),
+            type=FileType.DIRECTORY,
+            extra={"children_count": 0},
+        )
+    if resolved.entry is None:
+        raise enoent(path)
+    return FileStat(
+        name=resolved.entry.name,
+        type=FileType.TEXT,
+        size=resolved.entry.size,
+        modified=resolved.entry.extra.get("updated_at"),
+        fingerprint=None,
+        revision=None,
+        extra=dict(resolved.entry.extra),
+    )
+
+
+def stat_name(virtual_key: str, mount_prefix: str) -> str:
+    root = mount_prefix.rstrip("/") or "/"
+    if virtual_key == root:
+        return "/"
+    return virtual_key.rstrip("/").rsplit("/", 1)[-1]

python/mirage/core/chroma/tree.py

@@ -0,0 +1,161 @@
+import base64
+import gzip
+import json
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.chroma._client import fetch_path_tree
+from mirage.utils.path import gnu_basename, parent
+
+
+async def ensure_tree(accessor,
+                      index: IndexCacheStore,
+                      prefix: str = "") -> None:
+    root_key = mount_root(prefix)
+    listing = await index.list_dir(root_key)
+    if listing.entries is not None:
+        return
+
+    path_tree = parse_path_tree(await fetch_path_tree(accessor))
+    dir_entries = build_dir_entries(path_tree, prefix)
+    for directory in sorted(dir_entries):
+        await index.set_dir(
+            directory, sorted(dir_entries[directory],
+                              key=lambda item: item[0]))
+
+
+def parse_path_tree(raw: str) -> dict[str, dict[str, Any]]:
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        try:
+            decoded = gzip.decompress(base64.b64decode(raw)).decode()
+            parsed = json.loads(decoded)
+        except Exception as exc:
+            raise ValueError("Invalid Chroma path tree document") from exc
+    if not isinstance(parsed, dict):
+        raise ValueError("Chroma path tree must be a JSON object")
+    result: dict[str, dict[str, Any]] = {}
+    for key, value in parsed.items():
+        if not isinstance(key, str):
+            raise ValueError("Chroma path tree keys must be strings")
+        result[key] = value if isinstance(value, dict) else {}
+    return result
+
+
+def build_dir_entries(
+    path_tree: dict[str, dict[str, Any]],
+    prefix: str,
+) -> dict[str, list[tuple[str, IndexEntry]]]:
+    files: dict[str, dict[str, Any]] = {}
+    raw_slugs: dict[str, str] = {}
+    for raw_slug, metadata in path_tree.items():
+        path = normalize_slug(raw_slug)
+        if path in files:
+            value = path.strip("/")
+            raise ValueError(f"Duplicate Chroma path '{value}'")
+        files[path] = metadata
+        raw_slugs[path] = raw_slug
+
+    raise_on_collisions(files)
+    directories = collect_directories(set(files))
+    dir_entries: dict[str, list[tuple[str, IndexEntry]]] = {
+        virtual_path(directory, prefix): []
+        for directory in directories
+    }
+
+    for directory in sorted(directories):
+        if directory == "/":
+            continue
+        entry = IndexEntry(id=directory.strip("/"),
+                           name=gnu_basename(directory),
+                           resource_type="folder")
+        dir_entries[virtual_path(parent(directory), prefix)].append(
+            (entry.name, entry))
+
+    for path, metadata in sorted(files.items()):
+        slug = path.strip("/")
+        size = metadata_int_or_none(metadata, "size")
+        updated_at = metadata_or_none(metadata, "updated_at")
+        entry = IndexEntry(
+            id=slug,
+            name=gnu_basename(path),
+            resource_type="file",
+            size=size,
+            remote_time=updated_at or "",
+            extra={
+                "slug": slug,
+                "size": size,
+                "created_at": metadata_or_none(metadata, "created_at"),
+                "updated_at": updated_at,
+            },
+        )
+        dir_entries[virtual_path(parent(path), prefix)].append(
+            (entry.name, entry))
+    return dir_entries
+
+
+def normalize_slug(value: str) -> str:
+    parts = [part for part in value.strip("/").split("/") if part]
+    if not parts:
+        raise ValueError("Invalid empty Chroma path")
+    invalid = {".", ".."}
+    for part in parts:
+        if part in invalid:
+            raise ValueError(f"Invalid Chroma path segment: {part!r}")
+    return "/" + "/".join(parts)
+
+
+def raise_on_collisions(files: dict[str, dict[str, Any]]) -> None:
+    paths = set(files)
+    for path in sorted(paths):
+        parts = path.strip("/").split("/")
+        for index in range(1, len(parts)):
+            ancestor = "/" + "/".join(parts[:index])
+            if ancestor in paths:
+                raise ValueError(
+                    "Path collision: Chroma path "
+                    f"'{ancestor.strip('/')}' is both a file and a directory "
+                    f"prefix for '{path.strip()}'.")
+
+
+def collect_directories(paths: set[str]) -> set[str]:
+    directories = {"/"}
+    for path in paths:
+        parts = path.strip("/").split("/")
+        for index in range(1, len(parts)):
+            directories.add("/" + "/".join(parts[:index]))
+    return directories
+
+
+def metadata_or_none(metadata: dict[str, Any], key: str) -> str | None:
+    value = metadata.get(key)
+    if value is None:
+        return None
+    return str(value)
+
+
+def metadata_int_or_none(metadata: dict[str, Any], key: str) -> int | None:
+    value = metadata.get(key)
+    if isinstance(value, bool) or value is None:
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(value)
+    if isinstance(value, str) and value.isdigit():
+        return int(value)
+    return None
+
+
+def mount_root(prefix: str) -> str:
+    return prefix.rstrip("/") or "/"
+
+
+def virtual_path(path: str, prefix: str) -> str:
+    root = mount_root(prefix)
+    if path == "/":
+        return root
+    if root == "/":
+        return path
+    return root + path

python/mirage/core/chroma/walk.py

@@ -0,0 +1,49 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.chroma.path import resolve_path
+from mirage.core.chroma.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import rekey
+
+
+async def walk(
+    accessor,
+    path: PathSpec,
+    index: IndexCacheStore,
+    *,
+    include_root: bool = False,
+    maxdepth: int | None = None,
+    strip_prefix: bool = False,
+    ignore_missing: bool = False,
+    depth: int = 0,
+) -> list[str]:
+    try:
+        resolved = await resolve_path(accessor, path, index)
+    except (FileNotFoundError, NotADirectoryError):
+        if ignore_missing:
+            return []
+        raise
+
+    current = path.mount_path if strip_prefix else path.virtual
+    results = [current] if include_root else []
+    if not resolved.is_dir or (maxdepth is not None and depth >= maxdepth):
+        return results
+
+    try:
+        children = await readdir(accessor, path, index)
+    except (FileNotFoundError, NotADirectoryError):
+        if ignore_missing:
+            return results
+        raise
+
+    for child in children:
+        child_path = PathSpec.from_str_path(
+            child, rekey(path.virtual, path.resource_path, child))
+        results.extend(await walk(accessor,
+                                  child_path,
+                                  index,
+                                  include_root=True,
+                                  maxdepth=maxdepth,
+                                  strip_prefix=strip_prefix,
+                                  ignore_missing=ignore_missing,
+                                  depth=depth + 1))
+    return results

python/mirage/core/databricks_volume/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/mirage/core/databricks_volume/_helpers.py

@@ -0,0 +1,39 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+
+def ensure_path_spec(path: PathSpec | str) -> PathSpec:
+    if isinstance(path, PathSpec):
+        return path
+    return PathSpec.from_str_path(path)
+
+
+def parent_path(path: PathSpec | str) -> PathSpec:
+    path = ensure_path_spec(path)
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    stripped = path.mount_path.rstrip("/")
+    parent_relative = stripped.rsplit("/", 1)[0] if "/" in stripped else "/"
+    if not parent_relative.startswith("/"):
+        parent_relative = "/" + parent_relative
+    if prefix:
+        original = prefix.rstrip("/")
+        if parent_relative != "/":
+            original += parent_relative
+    else:
+        original = parent_relative
+    return PathSpec.from_str_path(original or "/",
+                                  mount_key(original or "/", prefix))

python/mirage/core/databricks_volume/copy.py

@@ -0,0 +1,111 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from io import BytesIO
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import ensure_path_spec
+from mirage.core.databricks_volume.path import backend_path
+from mirage.core.databricks_volume.read import read_bytes
+from mirage.core.databricks_volume.stat import stat
+from mirage.core.databricks_volume.write import write_bytes
+from mirage.types import FileType, PathSpec
+
+
+def _download_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> bytes:
+    response = accessor.files.download(remote_path)
+    contents = getattr(response, "contents", response)
+    if hasattr(contents, "read"):
+        return contents.read()
+    return bytes(contents)
+
+
+def _upload_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+    data: bytes,
+) -> None:
+    accessor.files.upload(remote_path, BytesIO(data), overwrite=True)
+
+
+def _create_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> None:
+    accessor.files.create_directory(remote_path)
+
+
+def _list_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> list:
+    return list(accessor.files.list_directory_contents(remote_path))
+
+
+def _copy_tree_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_src: str,
+    remote_dst: str,
+) -> None:
+    _create_directory_sync(accessor, remote_dst)
+    for entry in _list_directory_sync(accessor, remote_src):
+        name = entry.path.rstrip("/").rsplit("/", 1)[-1]
+        child_dst = remote_dst.rstrip("/") + "/" + name
+        if getattr(entry, "is_directory", False):
+            _copy_tree_sync(accessor, entry.path, child_dst)
+        else:
+            _upload_sync(accessor, child_dst,
+                         _download_sync(accessor, entry.path))
+
+
+async def copy(
+    accessor: DatabricksVolumeAccessor,
+    src: PathSpec,
+    dst: PathSpec,
+    index: IndexCacheStore = None,
+    recursive: bool = False,
+) -> None:
+    src = ensure_path_spec(src)
+    dst = ensure_path_spec(dst)
+    src_stat = await stat(accessor, src, index)
+    # Same-path guard runs after stat (and the non-recursive directory check)
+    # so a missing source or `cp` of a directory still raises.
+    same_path = backend_path(accessor.config,
+                             src) == backend_path(accessor.config, dst)
+    if src_stat.type == FileType.DIRECTORY:
+        if not recursive:
+            raise IsADirectoryError(src.virtual)
+        if same_path:
+            return
+        remote_src = backend_path(accessor.config, src)
+        remote_dst = backend_path(accessor.config, dst)
+        if remote_dst.startswith(remote_src + "/"):
+            # Copying a directory into its own subtree creates the destination
+            # inside the source, so the walk would descend into the fresh copy
+            # forever. Refuse before any create_directory/upload.
+            raise ValueError(f"cannot copy a directory, '{src.virtual}', "
+                             f"into itself, '{dst.virtual}'")
+        await asyncio.to_thread(_copy_tree_sync, accessor, remote_src,
+                                remote_dst)
+        return
+    if same_path:
+        # Copying a file onto itself would re-upload it; skip.
+        return
+    data = await read_bytes(accessor, src, index)
+    await write_bytes(accessor, dst, data, index)

python/mirage/core/databricks_volume/create.py

@@ -0,0 +1,26 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume.write import write_bytes
+from mirage.types import PathSpec
+
+
+async def create(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> None:
+    await write_bytes(accessor, path, b"", index)

python/mirage/core/databricks_volume/errors.py

@@ -0,0 +1,24 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+
+def is_not_found(exc: Exception) -> bool:
+    status_code = getattr(exc, "status_code", None)
+    if status_code == 404:
+        return True
+    error_code = getattr(exc, "error_code", None)
+    if error_code in {"RESOURCE_DOES_NOT_EXIST", "NOT_FOUND"}:
+        return True
+    message = str(exc).lower()
+    return "not found" in message or "does not exist" in message

python/mirage/core/databricks_volume/exists.py

@@ -0,0 +1,25 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.core.databricks_volume.stat import stat
+from mirage.types import PathSpec
+
+
+async def exists(accessor: DatabricksVolumeAccessor, path: PathSpec) -> bool:
+    try:
+        await stat(accessor, path)
+        return True
+    except FileNotFoundError:
+        return False

python/mirage/core/databricks_volume/glob.py

@@ -0,0 +1,29 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.constants import SCOPE_ERROR
+from mirage.core.databricks_volume.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: DatabricksVolumeAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/databricks_volume/mkdir.py

@@ -0,0 +1,60 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import (ensure_path_spec,
+                                                    parent_path)
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.exists import exists
+from mirage.core.databricks_volume.path import backend_path
+from mirage.core.databricks_volume.stat import stat
+from mirage.types import FileType, PathSpec
+from mirage.utils.errors import enoent, enotdir
+
+
+def _create_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> None:
+    accessor.files.create_directory(remote_path)
+
+
+async def mkdir(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+    parents: bool = False,
+) -> None:
+    path = ensure_path_spec(path)
+    remote_path = backend_path(accessor.config, path)
+    if parents:
+        await asyncio.to_thread(_create_directory_sync, accessor, remote_path)
+        return
+    if await exists(accessor, path):
+        raise FileExistsError(path.virtual)
+    parent = parent_path(path)
+    parent_stat = await stat(accessor, parent, index)
+    if parent_stat.type != FileType.DIRECTORY:
+        raise enotdir(path)
+    try:
+        await asyncio.to_thread(_create_directory_sync, accessor, remote_path)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    await invalidate_after_write(path)

python/mirage/core/databricks_volume/path.py

@@ -0,0 +1,72 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import posixpath
+
+from mirage.resource.databricks_volume.config import DatabricksVolumeConfig
+from mirage.types import PathSpec
+
+
+def volume_root(config: DatabricksVolumeConfig) -> str:
+    return posixpath.join("/Volumes", config.catalog, config.schema_name,
+                          config.volume)
+
+
+def _assert_inside_root(root: str, path: str, message: str) -> None:
+    if path == root or path.startswith(root + "/"):
+        return
+    raise ValueError(message)
+
+
+def configured_root(config: DatabricksVolumeConfig) -> str:
+    root_relative = config.root_path.strip("/")
+    if root_relative:
+        return posixpath.normpath(
+            posixpath.join(volume_root(config), root_relative))
+    return posixpath.normpath(volume_root(config))
+
+
+def backend_path(config: DatabricksVolumeConfig, path: PathSpec | str) -> str:
+    if isinstance(path, PathSpec):
+        raw = path.mount_path
+    else:
+        raw = path
+    relative = raw.strip("/")
+    root = configured_root(config)
+    parts = [root]
+    if relative:
+        parts.append(relative)
+    remote_path = posixpath.normpath(posixpath.join(*parts))
+    _assert_inside_root(
+        root,
+        remote_path,
+        f"path escapes Databricks volume root: {raw}",
+    )
+    return remote_path
+
+
+def virtual_path(config: DatabricksVolumeConfig,
+                 backend: str,
+                 prefix: str = "") -> str:
+    root = configured_root(config)
+    remote_path = posixpath.normpath(backend)
+    _assert_inside_root(
+        root,
+        remote_path,
+        f"backend path is outside Databricks volume root: {backend}",
+    )
+    relative = remote_path.removeprefix(root).strip("/")
+    path = "/" + relative if relative else "/"
+    return prefix.rstrip(
+        "/") + path if prefix and path != "/" else prefix or path

python/mirage/core/databricks_volume/read.py

@@ -0,0 +1,112 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import time
+from urllib.parse import quote
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path
+from mirage.observe.context import record
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+def _read_response_bytes(response) -> bytes:
+    if isinstance(response, dict):
+        contents = response.get("contents", response)
+    else:
+        contents = getattr(response, "contents", response)
+    if isinstance(contents, bytes):
+        return contents
+    if hasattr(contents, "read"):
+        return contents.read()
+    return bytes(contents)
+
+
+def _range_header(offset: int, size: int | None) -> str | None:
+    if offset < 0:
+        raise ValueError("offset must be non-negative")
+    if size is not None and size < 0:
+        raise ValueError("size must be non-negative")
+    if offset == 0 and size is None:
+        return None
+    if size is None:
+        return f"bytes={offset}-"
+    return f"bytes={offset}-{offset + size - 1}"
+
+
+def _download_bytes_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+    range_header: str | None,
+) -> bytes:
+    if range_header is None:
+        return _read_response_bytes(accessor.files.download(remote_path))
+    headers = {
+        "Accept": "application/octet-stream",
+        "Range": range_header,
+    }
+    cfg = getattr(accessor.client.api_client, "_cfg", None)
+    workspace_id = getattr(cfg, "workspace_id", None)
+    if workspace_id:
+        headers["X-Databricks-Org-Id"] = workspace_id
+    response = accessor.client.api_client.do(
+        "GET",
+        f"/api/2.0/fs/files{quote(remote_path)}",
+        headers=headers,
+        response_headers=[
+            "content-length",
+            "content-range",
+            "accept-ranges",
+            "content-type",
+            "last-modified",
+        ],
+        raw=True,
+    )
+    return _read_response_bytes(response)
+
+
+async def read_bytes(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+    offset: int = 0,
+    size: int | None = None,
+) -> bytes:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    remote_path = backend_path(accessor.config, path)
+    start_ms = int(time.monotonic() * 1000)
+    if size == 0:
+        record("read", virtual, "databricks_volume", 0, start_ms)
+        return b""
+    try:
+        data = await asyncio.to_thread(
+            _download_bytes_sync,
+            accessor,
+            remote_path,
+            _range_header(offset, size),
+        )
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    record("read", virtual, "databricks_volume", len(data), start_ms)
+    return data

python/mirage/core/databricks_volume/readdir.py

@@ -0,0 +1,90 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import logging
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path, virtual_path
+from mirage.core.databricks_volume.stat import modified_to_iso
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_prefix_of
+
+logger = logging.getLogger(__name__)
+SCOPE_ERROR = 10_000
+
+
+def _list_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> list[object]:
+    return list(accessor.files.list_directory_contents(remote_path))
+
+
+async def readdir(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore,
+) -> list[str]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    list_path = path.dir if path.pattern else path
+    virtual_key = list_path.virtual.rstrip("/") or "/"
+    listing = await index.list_dir(virtual_key)
+    if listing.entries is not None:
+        return listing.entries
+    remote_path = backend_path(accessor.config, list_path)
+    try:
+        entries = await asyncio.to_thread(
+            _list_directory_sync,
+            accessor,
+            remote_path,
+        )
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(list_path) from exc
+        raise
+    pairs = sorted(
+        (virtual_path(accessor.config, entry.path,
+                      mount_prefix_of(path.virtual, path.resource_path)),
+         entry) for entry in entries)
+    names = [name for name, _ in pairs]
+    if len(names) > SCOPE_ERROR:
+        logger.warning(
+            "databricks_volume readdir: %s returned %d entries (limit %d)",
+            virtual_key,
+            len(names),
+            SCOPE_ERROR,
+        )
+    index_entries = []
+    for full_path, entry in pairs:
+        name = full_path.rstrip("/").rsplit("/", 1)[-1]
+        resource_type = "folder" if getattr(entry, "is_directory",
+                                            False) else "file"
+        remote_time = modified_to_iso(getattr(entry, "last_modified", None))
+        index_entries.append((name,
+                              IndexEntry(
+                                  id=full_path,
+                                  name=name,
+                                  resource_type=resource_type,
+                                  size=getattr(entry, "file_size", None),
+                                  remote_time=remote_time or "",
+                              )))
+    await index.set_dir(virtual_key, index_entries)
+    return names

python/mirage/core/databricks_volume/rename.py

@@ -0,0 +1,60 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.context import (invalidate_after_unlink,
+                                  invalidate_after_write)
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import ensure_path_spec
+from mirage.core.databricks_volume.copy import copy
+from mirage.core.databricks_volume.path import backend_path
+from mirage.core.databricks_volume.rm import rm_recursive
+from mirage.core.databricks_volume.stat import stat
+from mirage.core.databricks_volume.unlink import unlink
+from mirage.types import FileType, PathSpec
+
+
+async def rename(
+    accessor: DatabricksVolumeAccessor,
+    src: PathSpec,
+    dst: PathSpec,
+    index: IndexCacheStore = None,
+) -> None:
+    # Non-atomic: the Databricks Files API has no native rename, so this is
+    # implemented as copy + delete and can leave partial state on failure.
+    src = ensure_path_spec(src)
+    dst = ensure_path_spec(dst)
+    src_stat = await stat(accessor, src, index)
+    remote_src = backend_path(accessor.config, src)
+    remote_dst = backend_path(accessor.config, dst)
+    if remote_src == remote_dst:
+        # rename(2) onto the same path is a no-op; copy + unlink here would
+        # upload the file onto itself then delete it, destroying the data.
+        # Guard runs after stat so a missing source still raises.
+        return
+    if src_stat.type == FileType.DIRECTORY:
+        if remote_dst.startswith(remote_src + "/"):
+            # Moving a directory into its own subtree would run away in the
+            # recursive copy and then rm_recursive would delete the original.
+            # Refuse before either side effect.
+            raise ValueError(
+                f"cannot move '{src.virtual}' to a subdirectory of "
+                f"itself, '{dst.virtual}'")
+        await copy(accessor, src, dst, index, recursive=True)
+        await rm_recursive(accessor, src, index)
+    else:
+        await copy(accessor, src, dst, index)
+        await unlink(accessor, src, index)
+    await invalidate_after_write(dst)
+    await invalidate_after_unlink(src)

python/mirage/core/databricks_volume/rm.py

@@ -0,0 +1,93 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import ensure_path_spec
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path, virtual_path
+from mirage.core.databricks_volume.stat import stat
+from mirage.core.databricks_volume.unlink import unlink
+from mirage.types import FileType, PathSpec
+from mirage.utils.errors import enoent
+
+
+def _list_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> list:
+    return list(accessor.files.list_directory_contents(remote_path))
+
+
+def _delete_file_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> None:
+    accessor.files.delete(remote_path)
+
+
+def _delete_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> None:
+    accessor.files.delete_directory(remote_path)
+
+
+def _remove_tree_recurse(
+    accessor: DatabricksVolumeAccessor,
+    remote_dir: str,
+    removed: list[str],
+) -> None:
+    for entry in _list_directory_sync(accessor, remote_dir):
+        if getattr(entry, "is_directory", False):
+            _remove_tree_recurse(accessor, entry.path, removed)
+        else:
+            _delete_file_sync(accessor, entry.path)
+            removed.append(entry.path)
+    _delete_directory_sync(accessor, remote_dir)
+    removed.append(remote_dir)
+
+
+def _remove_tree_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_root: str,
+) -> list[str]:
+    removed: list[str] = []
+    _remove_tree_recurse(accessor, remote_root, removed)
+    return removed
+
+
+async def rm_recursive(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> list[str]:
+    path = ensure_path_spec(path)
+    file_stat = await stat(accessor, path, index)
+    if file_stat.type != FileType.DIRECTORY:
+        await unlink(accessor, path, index)
+        return [path.mount_path]
+    remote_root = backend_path(accessor.config, path)
+    try:
+        removed = await asyncio.to_thread(_remove_tree_sync, accessor,
+                                          remote_root)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    await invalidate_after_unlink(path)
+    return [virtual_path(accessor.config, backend, "") for backend in removed]

python/mirage/core/databricks_volume/rmdir.py

@@ -0,0 +1,67 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import ensure_path_spec
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path
+from mirage.core.databricks_volume.stat import stat
+from mirage.types import FileType, PathSpec
+from mirage.utils.errors import enoent, enotdir
+
+
+def _list_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> list:
+    return list(accessor.files.list_directory_contents(remote_path))
+
+
+def _delete_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> None:
+    accessor.files.delete_directory(remote_path)
+
+
+async def rmdir(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> None:
+    path = ensure_path_spec(path)
+    file_stat = await stat(accessor, path, index)
+    if file_stat.type != FileType.DIRECTORY:
+        raise enotdir(path)
+    remote_path = backend_path(accessor.config, path)
+    try:
+        entries = await asyncio.to_thread(_list_directory_sync, accessor,
+                                          remote_path)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    if entries:
+        raise OSError(f"directory not empty: {path.virtual}")
+    try:
+        await asyncio.to_thread(_delete_directory_sync, accessor, remote_path)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    await invalidate_after_unlink(path)

python/mirage/core/databricks_volume/stat.py

@@ -0,0 +1,123 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.filetype import guess_type
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+def modified_to_iso(value) -> str | None:
+    if value is None or value == "":
+        return None
+    if isinstance(value, datetime):
+        if value.tzinfo is None:
+            value = value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc).isoformat()
+    if isinstance(value, (int, float)):
+        timestamp = value / 1000 if value > 10_000_000_000 else value
+        return datetime.fromtimestamp(timestamp, timezone.utc).isoformat()
+    try:
+        parsed = parsedate_to_datetime(str(value))
+    except (TypeError, ValueError):
+        return str(value)
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc).isoformat()
+
+
+def _is_directory(metadata) -> bool:
+    value = getattr(metadata, "is_directory", None)
+    if value is not None:
+        return bool(value)
+    object_type = getattr(metadata, "object_type", None)
+    if object_type is None:
+        return False
+    return str(object_type).lower().endswith("directory")
+
+
+def _name_from_backend_path(path: str) -> str:
+    return path.rstrip("/").rsplit("/", 1)[-1]
+
+
+async def _directory_stat_or_raise(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+    path: PathSpec,
+) -> FileStat:
+    try:
+        await asyncio.to_thread(accessor.files.get_directory_metadata,
+                                remote_path)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    return FileStat(name=_name_from_backend_path(remote_path),
+                    type=FileType.DIRECTORY)
+
+
+async def stat(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore | None = None,
+) -> FileStat:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    stripped = path.mount_path.strip("/")
+    if not stripped:
+        return FileStat(name="/", type=FileType.DIRECTORY)
+    if index is not None:
+        prefix = mount_prefix_of(path.virtual, path.resource_path)
+        virtual_key = (prefix.rstrip("/") + "/" + stripped if prefix else "/" +
+                       stripped)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is not None:
+            entry = lookup.entry
+            if entry.resource_type == "folder":
+                return FileStat(name=entry.name, type=FileType.DIRECTORY)
+            return FileStat(name=entry.name,
+                            size=entry.size,
+                            modified=entry.remote_time or None,
+                            type=guess_type(entry.name))
+        parent = virtual_key.rsplit("/", 1)[0] or "/"
+        parent_listing = await index.list_dir(parent)
+        if parent_listing.entries is not None:
+            raise enoent(path)
+    remote_path = backend_path(accessor.config, path)
+    try:
+        metadata = await asyncio.to_thread(accessor.files.get_metadata,
+                                           remote_path)
+    except Exception as exc:
+        if is_not_found(exc):
+            return await _directory_stat_or_raise(accessor, remote_path, path)
+        raise
+    name = _name_from_backend_path(remote_path)
+    if _is_directory(metadata):
+        return FileStat(name=name, type=FileType.DIRECTORY)
+    size = getattr(metadata, "content_length", None)
+    modified = modified_to_iso(getattr(metadata, "last_modified", None))
+    return FileStat(name=name,
+                    size=size,
+                    modified=modified,
+                    type=guess_type(name))

python/mirage/core/databricks_volume/stream.py

@@ -0,0 +1,89 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from collections.abc import AsyncIterator
+from typing import BinaryIO
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path
+from mirage.core.databricks_volume.read import read_bytes
+from mirage.observe.context import record_stream
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+def _download_contents(response) -> BinaryIO:
+    if isinstance(response, dict):
+        contents = response.get("contents")
+    else:
+        contents = getattr(response, "contents", None)
+    if contents is None:
+        raise RuntimeError("Databricks download response has no contents")
+    return contents
+
+
+def _open_download_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> BinaryIO:
+    return _download_contents(accessor.files.download(remote_path))
+
+
+async def read_stream(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+    chunk_size: int = 8192,
+) -> AsyncIterator[bytes]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if chunk_size <= 0:
+        raise ValueError("chunk_size must be positive")
+    rec = record_stream("read", path.virtual, "databricks_volume")
+    remote_path = backend_path(accessor.config, path)
+    contents = None
+    try:
+        contents = await asyncio.to_thread(
+            _open_download_sync,
+            accessor,
+            remote_path,
+        )
+        while True:
+            chunk = await asyncio.to_thread(contents.read, chunk_size)
+            if not chunk:
+                return
+            if rec is not None:
+                rec.bytes += len(chunk)
+            yield chunk
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    finally:
+        if contents is not None:
+            await asyncio.to_thread(contents.close)
+
+
+async def range_read(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    start: int,
+    end: int,
+) -> bytes:
+    return await read_bytes(accessor, path, offset=start, size=end - start)

python/mirage/core/databricks_volume/unlink.py

@@ -0,0 +1,55 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import time
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import ensure_path_spec
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path
+from mirage.core.databricks_volume.stat import stat
+from mirage.observe.context import record
+from mirage.types import FileType, PathSpec
+from mirage.utils.errors import enoent
+
+
+def _delete_file_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+) -> None:
+    accessor.files.delete(remote_path)
+
+
+async def unlink(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> None:
+    path = ensure_path_spec(path)
+    file_stat = await stat(accessor, path, index)
+    if file_stat.type == FileType.DIRECTORY:
+        raise IsADirectoryError(path.virtual)
+    remote_path = backend_path(accessor.config, path)
+    start_ms = int(time.monotonic() * 1000)
+    try:
+        await asyncio.to_thread(_delete_file_sync, accessor, remote_path)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    record("unlink", path.virtual, "databricks_volume", 0, start_ms)
+    await invalidate_after_unlink(path)

python/mirage/core/databricks_volume/write.py

@@ -0,0 +1,97 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import time
+from io import BytesIO
+
+from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.cache.index import IndexCacheStore
+from mirage.core.databricks_volume._helpers import (ensure_path_spec,
+                                                    parent_path)
+from mirage.core.databricks_volume.errors import is_not_found
+from mirage.core.databricks_volume.path import backend_path
+from mirage.observe.context import record
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+def _is_directory_metadata(metadata: object) -> bool:
+    value = getattr(metadata, "is_directory", None)
+    if value is not None:
+        return bool(value)
+    object_type = getattr(metadata, "object_type", None)
+    if object_type is None:
+        return False
+    return str(object_type).lower().endswith("directory")
+
+
+def _ensure_parent_directory_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_parent: str,
+    virtual_target: str,
+) -> None:
+    try:
+        accessor.files.get_directory_metadata(remote_parent)
+        return
+    except Exception as exc:
+        if not is_not_found(exc):
+            raise
+        not_found = exc
+    try:
+        metadata = accessor.files.get_metadata(remote_parent)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise FileNotFoundError(virtual_target) from not_found
+        raise
+    if not _is_directory_metadata(metadata):
+        raise NotADirectoryError(virtual_target)
+
+
+def _upload_bytes_sync(
+    accessor: DatabricksVolumeAccessor,
+    remote_path: str,
+    data: bytes,
+) -> None:
+    accessor.files.upload(remote_path, BytesIO(data), overwrite=True)
+
+
+async def write_bytes(
+    accessor: DatabricksVolumeAccessor,
+    path: PathSpec,
+    data: bytes,
+    index: IndexCacheStore = None,
+) -> None:
+    path = ensure_path_spec(path)
+    parent = parent_path(path)
+    remote_parent = backend_path(accessor.config, parent)
+    remote_path = backend_path(accessor.config, path)
+    start_ms = int(time.monotonic() * 1000)
+    # TODO native async client calling HTTP API as databricks sdk is sync
+    await asyncio.to_thread(
+        _ensure_parent_directory_sync,
+        accessor,
+        remote_parent,
+        path.virtual,
+    )
+    try:
+        await asyncio.to_thread(_upload_bytes_sync, accessor, remote_path,
+                                data)
+    except Exception as exc:
+        if is_not_found(exc):
+            raise enoent(path) from exc
+        raise
+    record("write", path.virtual, "databricks_volume", len(data), start_ms)
+    await invalidate_after_write(path)

python/mirage/core/dify/__init__.py

@@ -0,0 +1 @@
+__all__ = []

python/mirage/core/dify/_client.py

@@ -0,0 +1,125 @@
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from typing import Any
+
+import httpx
+
+from mirage.resource.dify.config import DifyConfig
+
+logger = logging.getLogger(__name__)
+
+
+async def dify_get(config: DifyConfig,
+                   endpoint: str,
+                   params: dict[str, Any] | None = None) -> dict[str, Any]:
+    url = f"{config.base_url}{endpoint}"
+    headers = {"Authorization": f"Bearer {config.api_key}"}
+    last_error: httpx.HTTPStatusError | None = None
+    for attempt in range(4):
+        async with httpx.AsyncClient(timeout=30) as client:
+            response = await client.get(url, headers=headers, params=params)
+        if response.status_code == 429 and attempt < 3:
+            logger.warning("Dify rate limited request to %s", endpoint)
+            await asyncio.sleep(2**attempt)
+            continue
+        if 500 <= response.status_code < 600 and attempt < 1:
+            await asyncio.sleep(1)
+            continue
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            last_error = exc
+            break
+        return response.json()
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"Dify request failed: {endpoint}")
+
+
+async def dify_post(config: DifyConfig, endpoint: str,
+                    body: dict[str, Any]) -> dict[str, Any]:
+    url = f"{config.base_url}{endpoint}"
+    headers = {"Authorization": f"Bearer {config.api_key}"}
+    last_error: httpx.HTTPStatusError | None = None
+    for attempt in range(4):
+        async with httpx.AsyncClient(timeout=30) as client:
+            response = await client.post(url, headers=headers, json=body)
+        if response.status_code == 429 and attempt < 3:
+            logger.warning("Dify rate limited request to %s", endpoint)
+            await asyncio.sleep(2**attempt)
+            continue
+        if 500 <= response.status_code < 600 and attempt < 1:
+            await asyncio.sleep(1)
+            continue
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            last_error = exc
+            break
+        return response.json()
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"Dify request failed: {endpoint}")
+
+
+async def list_all_documents(config: DifyConfig) -> list[dict[str, Any]]:
+    documents: list[dict[str, Any]] = []
+    page = 1
+    while True:
+        payload = await dify_get(
+            config,
+            f"/datasets/{config.dataset_id}/documents",
+            {
+                "page": page,
+                "limit": 100
+            },
+        )
+        for document in payload.get("data") or []:
+            if is_visible_document(document):
+                documents.append(document)
+        if not payload.get("has_more"):
+            return documents
+        page += 1
+
+
+async def get_document_detail(config: DifyConfig,
+                              document_id: str) -> dict[str, Any]:
+    return await dify_get(
+        config, f"/datasets/{config.dataset_id}/documents/{document_id}")
+
+
+async def get_document_segments(config: DifyConfig,
+                                document_id: str) -> list[dict[str, Any]]:
+    segments: list[dict[str, Any]] = []
+    async for page in iter_segment_pages(config, document_id):
+        segments.extend(page)
+    return segments
+
+
+async def iter_segment_pages(
+    config: DifyConfig,
+    document_id: str,
+) -> AsyncIterator[list[dict[str, Any]]]:
+    page = 1
+    while True:
+        payload = await dify_get(
+            config,
+            f"/datasets/{config.dataset_id}/documents/{document_id}/segments",
+            {
+                "page": page,
+                "limit": 100,
+                "status": "completed",
+                "enabled": "true",
+            },
+        )
+        yield payload.get("data") or []
+        if not payload.get("has_more"):
+            return
+        page += 1
+
+
+def is_visible_document(document: dict[str, Any]) -> bool:
+    return (document.get("enabled") is True
+            and document.get("indexing_status") == "completed"
+            and document.get("archived") is False)

python/mirage/core/dify/find.py

@@ -0,0 +1,113 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
+                                               keep, start_basename,
+                                               tree_has_type)
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.stat import stat
+from mirage.core.dify.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+
+async def find(
+    accessor,
+    path: PathSpec,
+    name: str | None = None,
+    type: str | None = None,
+    min_size: int | None = None,
+    max_size: int | None = None,
+    maxdepth: int | None = None,
+    name_exclude: str | None = None,
+    or_names: list[str] | None = None,
+    mtime_min: float | None = None,
+    mtime_max: float | None = None,
+    iname: str | None = None,
+    path_pattern: str | None = None,
+    mindepth: int | None = None,
+    empty: bool = False,
+    tree: PredNode | None = None,
+    index: IndexCacheStore | None = None,
+) -> list[str]:
+    if index is None:
+        raise ValueError("find: missing index")
+    results = await walk(accessor,
+                         path,
+                         index,
+                         include_root=True,
+                         maxdepth=maxdepth,
+                         strip_prefix=True)
+    tree = tree if tree is not None else build_tree(name=name,
+                                                    iname=iname,
+                                                    path_pattern=path_pattern,
+                                                    type=type,
+                                                    name_exclude=name_exclude,
+                                                    or_names=or_names)
+    needs_kind = (tree_has_type(tree) or min_size is not None
+                  or max_size is not None)
+    start_name = start_basename(path)
+    filtered: list[str] = []
+    for item in results:
+        if await _matches(accessor, item,
+                          mount_prefix_of(path.virtual, path.resource_path),
+                          index, path.mount_path, tree, needs_kind, min_size,
+                          max_size, mindepth, start_name):
+            filtered.append(item)
+    return sorted(filtered)
+
+
+async def _matches(
+    accessor,
+    item: str,
+    prefix: str,
+    index: IndexCacheStore,
+    root: str,
+    tree: PredNode,
+    needs_kind: bool,
+    min_size: int | None,
+    max_size: int | None,
+    mindepth: int | None,
+    start_name: str,
+) -> bool:
+    root_norm = root.rstrip("/") or "/"
+    item_norm = item.rstrip("/") or "/"
+    item_name = (start_name if item_norm == root_norm else
+                 item.rstrip("/").rsplit("/", 1)[-1])
+    spec = PathSpec.from_str_path(item, mount_key(item, prefix))
+    kind = "f"
+    if needs_kind:
+        resolved = await resolve_path(accessor, spec, index)
+        kind = "d" if resolved.is_dir else "f"
+    entry = FindEntry(key=item,
+                      name=item_name,
+                      kind=kind,
+                      depth=_relative_depth(item, root))
+    if not keep(entry, tree, mindepth):
+        return False
+    # Directories count as size 0 for -size (deliberate GNU divergence).
+    if min_size is not None or max_size is not None:
+        if kind == "d":
+            size = 0
+        else:
+            item_stat = await stat(accessor, spec, index)
+            if item_stat.size is None:
+                return False
+            size = item_stat.size
+        if min_size is not None and size < min_size:
+            return False
+        if max_size is not None and size > max_size:
+            return False
+    return True
+
+
+def _relative_depth(item: str, root: str) -> int:
+    root_norm = root.rstrip("/") or "/"
+    item_norm = item.rstrip("/") or "/"
+    if item_norm == root_norm:
+        return 0
+    if root_norm == "/":
+        relative = item_norm.strip("/")
+    else:
+        relative = item_norm.removeprefix(root_norm).lstrip("/")
+    if not relative:
+        return 0
+    return relative.count("/") + 1

python/mirage/core/dify/glob.py

@@ -0,0 +1,15 @@
+from mirage.accessor.dify import DifyAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.constants import SCOPE_ERROR
+from mirage.core.dify.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: DifyAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/dify/grep.py

@@ -0,0 +1,40 @@
+import re
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify.read import read_stream
+from mirage.io.async_line_iterator import AsyncLineIterator
+from mirage.types import PathSpec
+
+
+async def grep_bytes(
+        accessor,
+        paths: list[PathSpec],
+        pattern: str,
+        index: IndexCacheStore,
+        ignore_case: bool = False) -> tuple[bytes, dict[str, bytes]]:
+    flags = re.IGNORECASE if ignore_case else 0
+    regex = re.compile(pattern, flags)
+    lines: list[str] = []
+    reads: dict[str, bytes] = {}
+    for path in paths:
+        chunks: list[bytes] = []
+        stream = _record_chunks(read_stream(accessor, path, index), chunks)
+        async for line_number, raw_line in _enumerate_lines(stream):
+            line = raw_line.decode(errors="replace")
+            if regex.search(line):
+                lines.append(f"{path.virtual}:{line_number}:{line}")
+        reads[path.virtual] = b"".join(chunks)
+    return "\n".join(lines).encode(), reads
+
+
+async def _record_chunks(source, chunks: list[bytes]):
+    async for chunk in source:
+        chunks.append(chunk)
+        yield chunk
+
+
+async def _enumerate_lines(source):
+    line_number = 0
+    async for raw_line in AsyncLineIterator(source):
+        line_number += 1
+        yield line_number, raw_line

python/mirage/core/dify/path.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.dify.tree import ensure_tree
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+@dataclass(frozen=True)
+class ResolvedDifyPath:
+    virtual_key: str
+    mount_prefix: str
+    is_dir: bool
+    entry: IndexEntry | None = None
+
+
+async def resolve_path(accessor, path: PathSpec,
+                       index: IndexCacheStore) -> ResolvedDifyPath:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    mount_prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+    await ensure_tree(accessor, index, mount_prefix)
+    virtual_key = virtual_key_for(path)
+    result = await index.get(virtual_key)
+    if result.entry is not None:
+        return ResolvedDifyPath(
+            virtual_key=virtual_key,
+            mount_prefix=mount_prefix,
+            is_dir=result.entry.resource_type == "folder",
+            entry=result.entry,
+        )
+    listing = await index.list_dir(virtual_key)
+    if listing.entries is not None:
+        return ResolvedDifyPath(virtual_key=virtual_key,
+                                mount_prefix=mount_prefix,
+                                is_dir=True)
+    raise enoent(path)
+
+
+def virtual_key_for(path: PathSpec) -> str:
+    raw = path.directory if path.pattern else path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+    if prefix:
+        root = prefix.rstrip("/") or "/"
+        if raw == root or raw.startswith(root + "/"):
+            return raw.rstrip("/") or root
+        rest = raw.strip("/")
+        if not rest:
+            return root
+        return root + "/" + rest
+    stripped = raw.strip("/")
+    return "/" + stripped if stripped else "/"

python/mirage/core/dify/read.py

@@ -0,0 +1,45 @@
+import errno
+from collections.abc import AsyncIterator
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify._client import get_document_segments, iter_segment_pages
+from mirage.core.dify.path import resolve_path
+from mirage.types import PathSpec
+
+
+async def read_bytes(accessor, path: PathSpec,
+                     index: IndexCacheStore) -> bytes:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
+    segments = await get_document_segments(accessor.config, resolved.entry.id)
+    return segments_to_bytes(segments)
+
+
+async def read_stream(accessor, path: PathSpec,
+                      index: IndexCacheStore) -> AsyncIterator[bytes]:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        raise IsADirectoryError(errno.EISDIR, "Is a directory", path.virtual)
+    first = True
+    async for page in iter_segment_pages(accessor.config, resolved.entry.id):
+        for segment in page:
+            if first:
+                first = False
+            else:
+                yield b"\n"
+            yield segment_text(segment).encode()
+
+
+def segments_to_bytes(segments: list[dict[str, Any]]) -> bytes:
+    return "\n".join(segment_text(segment) for segment in segments).encode()
+
+
+def segment_text(segment: dict[str, Any]) -> str:
+    value = segment.get("content")
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return str(value)

python/mirage/core/dify/readdir.py

@@ -0,0 +1,15 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify.path import resolve_path
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent, enotdir
+
+
+async def readdir(accessor, path: PathSpec,
+                  index: IndexCacheStore) -> list[str]:
+    resolved = await resolve_path(accessor, path, index)
+    if not resolved.is_dir:
+        raise enotdir(path)
+    listing = await index.list_dir(resolved.virtual_key)
+    if listing.entries is None:
+        raise enoent(path)
+    return listing.entries

python/mirage/core/dify/search.py

@@ -0,0 +1,224 @@
+import logging
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.dify._client import dify_post
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.read import segment_text
+from mirage.core.dify.tree import normalize_slug
+from mirage.core.dify.walk import walk
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_prefix_of, rekey
+from mirage.utils.score import format_score
+
+logger = logging.getLogger(__name__)
+
+METHODS = {
+    "semantic": "semantic_search",
+    "fulltext": "full_text_search",
+    "hybrid": "hybrid_search",
+    "keyword": "keyword_search",
+}
+
+
+async def search_segments(
+    accessor,
+    query: str,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+    method: str = "semantic",
+    top_k: int = 10,
+    threshold: float = 0.0,
+    mount_prefix: str = "",
+) -> bytes:
+    search_method = validate_args(query, method, top_k, threshold)
+    if not mount_prefix and paths:
+        mount_prefix = mount_prefix_of(paths[0].virtual,
+                                       paths[0].resource_path)
+    retrieval_model = {
+        "search_method": search_method,
+        "top_k": min(top_k, 100),
+        "score_threshold_enabled": threshold > 0,
+        "score_threshold": threshold,
+        "reranking_enable": False,
+    }
+    has_name_based_target = False
+    if paths:
+        conditions, has_name_based_target = await metadata_conditions(
+            accessor, paths, index)
+        if not conditions:
+            return b""
+        retrieval_model["metadata_filtering_conditions"] = {
+            "logical_operator": "or",
+            "conditions": conditions,
+        }
+    response = await dify_post(
+        accessor.config,
+        f"/datasets/{accessor.config.dataset_id}/retrieve",
+        {
+            "query": query,
+            "retrieval_model": retrieval_model
+        },
+    )
+    output = records_to_bytes(
+        response.get("records") or [], accessor.config.slug_metadata_name,
+        mount_prefix)
+    if paths and has_name_based_target and output == b"":
+        logger.debug(
+            "Dify scoped search returned no records for name-based documents; "
+            "check that Built-in Fields are enabled in Dify dataset metadata.")
+    return output
+
+
+def validate_args(query: str, method: str, top_k: int,
+                  threshold: float) -> str:
+    if not query:
+        raise ValueError("search: query is required")
+    if len(query) > 250:
+        raise ValueError("search: query cannot exceed 250 characters")
+    if top_k <= 0:
+        raise ValueError("search: top-k must be positive")
+    if threshold < 0 or threshold > 1:
+        raise ValueError("search: threshold must be in [0, 1]")
+    if method not in METHODS:
+        raise ValueError(
+            "search: method must be one of semantic, fulltext, hybrid, keyword"
+        )
+    return METHODS[method]
+
+
+async def metadata_conditions(
+    accessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> tuple[list[dict], bool]:
+    targets = await target_entries(accessor, paths, index)
+    slug_values: list[str] = []
+    name_values: list[str] = []
+    for entry in targets.values():
+        if entry.extra.get("has_slug") is True:
+            slug_values.append(str(entry.extra["raw_slug"]))
+        else:
+            name_values.append(entry.name)
+    conditions: list[dict] = []
+    if slug_values:
+        conditions.append({
+            "name": accessor.config.slug_metadata_name,
+            "comparison_operator": "in",
+            "value": sorted(slug_values),
+        })
+    if name_values:
+        conditions.append({
+            "name": "document_name",
+            "comparison_operator": "in",
+            "value": sorted(name_values),
+        })
+    return conditions, bool(name_values)
+
+
+async def target_entries(
+    accessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> dict[str, IndexEntry]:
+    targets: dict[str, IndexEntry] = {}
+    for path in paths:
+        resolved = await resolve_path(accessor, path, index)
+        if resolved.entry is not None and not resolved.is_dir:
+            targets[resolved.entry.id] = resolved.entry
+            continue
+        if resolved.is_dir:
+            children = await walk(accessor,
+                                  path,
+                                  index,
+                                  include_root=False,
+                                  strip_prefix=False)
+            for child in children:
+                child_spec = PathSpec.from_str_path(
+                    child, rekey(path.virtual, path.resource_path, child))
+                child_resolved = await resolve_path(accessor, child_spec,
+                                                    index)
+                if (child_resolved.entry is not None
+                        and not child_resolved.is_dir):
+                    targets[child_resolved.entry.id] = child_resolved.entry
+    return targets
+
+
+def records_to_bytes(
+    records: list[dict[str, Any]],
+    slug_metadata_name: str,
+    mount_prefix: str,
+) -> bytes:
+    contents: list[str] = []
+    for record in records:
+        segment = record.get("segment")
+        if not isinstance(segment, dict):
+            continue
+        header = format_record_header(record, slug_metadata_name, mount_prefix)
+        if header is None:
+            continue
+        content = segment_text(segment)
+        contents.append(f"{header}\n{content}")
+    if not contents:
+        return b""
+    return ("\n".join(contents) + "\n").encode()
+
+
+def format_record_header(
+    record: dict[str, Any],
+    slug_metadata_name: str,
+    mount_prefix: str,
+) -> str | None:
+    path = record_path(record, slug_metadata_name, mount_prefix)
+    if path is None:
+        return None
+    score = format_score(record.get("score"))
+    if score is None:
+        return path
+    return f"{path}:{score}"
+
+
+def record_path(
+    record: dict[str, Any],
+    slug_metadata_name: str,
+    mount_prefix: str,
+) -> str | None:
+    segment = record.get("segment")
+    if not isinstance(segment, dict):
+        return None
+    document = segment.get("document")
+    if not isinstance(document, dict):
+        return None
+    raw_path = document_path(document, slug_metadata_name)
+    if raw_path is None:
+        return None
+    try:
+        normalized = normalize_slug(raw_path)
+    except ValueError:
+        logger.debug("Skipping Dify record with invalid slug/name: %r",
+                     raw_path)
+        return None
+    prefix = mount_prefix.rstrip("/")
+    if not prefix:
+        return normalized
+    return prefix + normalized
+
+
+def document_path(
+    document: dict[str, Any],
+    slug_metadata_name: str,
+) -> str | None:
+    metadata = document.get("doc_metadata")
+    if isinstance(metadata, list):
+        for item in metadata:
+            if (isinstance(item, dict)
+                    and item.get("name") == slug_metadata_name
+                    and item.get("value") is not None):
+                return str(item["value"])
+    if isinstance(metadata,
+                  dict) and metadata.get(slug_metadata_name) is not None:
+        return str(metadata[slug_metadata_name])
+    name = document.get("name")
+    if name is None:
+        return None
+    return str(name)

python/mirage/core/dify/stat.py

@@ -0,0 +1,77 @@
+from datetime import datetime, timezone
+
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify._client import get_document_detail
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.tree import extract_document_size
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+
+
+async def stat_light(accessor, path: PathSpec,
+                     index: IndexCacheStore) -> FileStat:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        return FileStat(
+            name=stat_name(resolved.virtual_key, resolved.mount_prefix),
+            type=FileType.DIRECTORY,
+            extra={"children_count": 0},
+        )
+    if resolved.entry is None:
+        raise enoent(path)
+    return FileStat(
+        name=resolved.entry.name,
+        type=FileType.TEXT,
+        size=resolved.entry.size,
+        modified=timestamp_to_zulu(resolved.entry.remote_time),
+        fingerprint=None,
+        revision=None,
+        extra=dict(resolved.entry.extra),
+    )
+
+
+async def stat(accessor, path: PathSpec, index: IndexCacheStore) -> FileStat:
+    resolved = await resolve_path(accessor, path, index)
+    if resolved.is_dir:
+        return FileStat(
+            name=stat_name(resolved.virtual_key, resolved.mount_prefix),
+            type=FileType.DIRECTORY,
+            extra={"children_count": 0},
+        )
+    if resolved.entry is None:
+        raise enoent(path)
+    detail = await get_document_detail(accessor.config, resolved.entry.id)
+    size = extract_document_size(detail)
+    if size is None:
+        size = resolved.entry.size
+    extra = dict(resolved.entry.extra)
+    extra["document_id"] = resolved.entry.id
+    if "tokens" in detail:
+        extra["tokens"] = detail.get("tokens")
+    if "indexing_status" in detail:
+        extra["indexing_status"] = detail.get("indexing_status")
+    return FileStat(
+        name=resolved.entry.name,
+        type=FileType.TEXT,
+        size=size,
+        modified=timestamp_to_zulu(detail.get("updated_at")),
+        fingerprint=None,
+        revision=None,
+        extra=extra,
+    )
+
+
+def timestamp_to_zulu(value: object) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        return datetime.fromtimestamp(
+            value, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    return str(value)
+
+
+def stat_name(virtual_key: str, mount_prefix: str) -> str:
+    root = mount_prefix.rstrip("/") or "/"
+    if virtual_key == root:
+        return "/"
+    return virtual_key.rstrip("/").rsplit("/", 1)[-1]

python/mirage/core/dify/tree.py

@@ -0,0 +1,174 @@
+from datetime import datetime, timezone
+from typing import Any
+
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.dify._client import list_all_documents
+from mirage.utils.path import gnu_basename, parent
+
+
+async def ensure_tree(accessor,
+                      index: IndexCacheStore,
+                      prefix: str = "") -> None:
+    root_key = mount_root(prefix)
+    listing = await index.list_dir(root_key)
+    if listing.entries is not None:
+        return
+
+    # list_all_documents already filters to visible documents.
+    documents = await list_all_documents(accessor.config)
+    dir_entries = build_dir_entries(
+        documents,
+        prefix,
+        accessor.config.slug_metadata_name,
+    )
+    for directory in sorted(dir_entries):
+        await index.set_dir(
+            directory, sorted(dir_entries[directory],
+                              key=lambda item: item[0]))
+
+
+def build_dir_entries(
+    documents: list[dict[str, Any]],
+    prefix: str,
+    slug_metadata_name: str = "slug",
+) -> dict[str, list[tuple[str, IndexEntry]]]:
+    files: dict[str, dict[str, Any]] = {}
+    raw_slugs: dict[str, str] = {}
+    has_slugs: dict[str, bool] = {}
+    for document in documents:
+        slug, has_slug = extract_slug(document, slug_metadata_name)
+        path = normalize_slug(slug)
+        if path in files:
+            value = path.strip("/")
+            raise ValueError(
+                f"Duplicate {slug_metadata_name} '{value}': documents "
+                f"'{files[path].get('id')}' and '{document.get('id')}' share "
+                "the same path.")
+        files[path] = document
+        raw_slugs[path] = str(slug)
+        has_slugs[path] = has_slug
+
+    raise_on_collisions(files)
+    directories = collect_directories(set(files))
+    dir_entries: dict[str, list[tuple[str, IndexEntry]]] = {
+        virtual_path(directory, prefix): []
+        for directory in directories
+    }
+    for directory in sorted(directories):
+        if directory == "/":
+            continue
+        entry = IndexEntry(
+            id=directory.strip("/"),
+            name=gnu_basename(directory),
+            resource_type="folder",
+        )
+        dir_entries[virtual_path(parent(directory), prefix)].append(
+            (entry.name, entry))
+
+    for path, document in sorted(files.items()):
+        entry = IndexEntry(
+            id=str(document["id"]),
+            name=gnu_basename(path),
+            resource_type="file",
+            size=extract_document_size(document),
+            remote_time=timestamp_to_iso(document.get("created_at")),
+            extra={
+                "slug": path.strip("/"),
+                "slug_metadata_name": slug_metadata_name,
+                "raw_slug": raw_slugs[path],
+                "has_slug": has_slugs[path],
+                "tokens": document.get("tokens"),
+                "indexing_status": document.get("indexing_status"),
+                "data_source_type": document.get("data_source_type"),
+            },
+        )
+        dir_entries[virtual_path(parent(path), prefix)].append(
+            (entry.name, entry))
+    return dir_entries
+
+
+def extract_slug(document: dict[str, Any],
+                 slug_metadata_name: str = "slug") -> tuple[str, bool]:
+    metadata = document.get("doc_metadata")
+    if isinstance(metadata, list):
+        for item in metadata:
+            if (isinstance(item, dict)
+                    and item.get("name") == slug_metadata_name):
+                value = item.get("value")
+                if value is not None:
+                    return str(value), True
+    if (isinstance(metadata, dict)
+            and metadata.get(slug_metadata_name) is not None):
+        return str(metadata[slug_metadata_name]), True
+    return str(document["name"]), False
+
+
+def normalize_slug(value: str) -> str:
+    parts = [part for part in value.strip("/").split("/") if part]
+    if not parts:
+        raise ValueError("Invalid empty Dify document slug.")
+    invalid = {".", ".."}
+    for part in parts:
+        if part in invalid:
+            raise ValueError(f"Invalid Dify document slug segment: {part!r}")
+    return "/" + "/".join(parts)
+
+
+def raise_on_collisions(files: dict[str, dict[str, Any]]) -> None:
+    paths = set(files)
+    for path in sorted(paths):
+        parts = path.strip("/").split("/")
+        for index in range(1, len(parts)):
+            ancestor = "/" + "/".join(parts[:index])
+            if ancestor in paths:
+                raise ValueError(
+                    "Path collision: document "
+                    f"'{files[ancestor].get('id')}' uses file path "
+                    f"'{ancestor.strip('/')}' but document "
+                    f"'{files[path].get('id')}' requires it as a directory "
+                    "prefix.")
+
+
+def collect_directories(paths: set[str]) -> set[str]:
+    directories = {"/"}
+    for path in paths:
+        parts = path.strip("/").split("/")
+        for index in range(1, len(parts)):
+            directories.add("/" + "/".join(parts[:index]))
+    return directories
+
+
+def extract_document_size(document: dict[str, Any]) -> int | None:
+    candidates = (
+        document.get("data_source_detail_dict"),
+        document.get("data_source_info"),
+    )
+    for candidate in candidates:
+        if isinstance(candidate, dict):
+            upload_file = candidate.get("upload_file")
+            if isinstance(upload_file, dict):
+                size = upload_file.get("size")
+                if isinstance(size, int):
+                    return size
+    return None
+
+
+def timestamp_to_iso(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, (int, float)):
+        return datetime.fromtimestamp(value, timezone.utc).isoformat()
+    return str(value)
+
+
+def mount_root(prefix: str) -> str:
+    return prefix.rstrip("/") or "/"
+
+
+def virtual_path(path: str, prefix: str) -> str:
+    root = mount_root(prefix)
+    if path == "/":
+        return root
+    if root == "/":
+        return path
+    return root + path

python/mirage/core/dify/walk.py

@@ -0,0 +1,49 @@
+from mirage.cache.index import IndexCacheStore
+from mirage.core.dify.path import resolve_path
+from mirage.core.dify.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import rekey
+
+
+async def walk(
+    accessor,
+    path: PathSpec,
+    index: IndexCacheStore,
+    *,
+    include_root: bool = False,
+    maxdepth: int | None = None,
+    strip_prefix: bool = False,
+    ignore_missing: bool = False,
+    depth: int = 0,
+) -> list[str]:
+    try:
+        resolved = await resolve_path(accessor, path, index)
+    except (FileNotFoundError, NotADirectoryError):
+        if ignore_missing:
+            return []
+        raise
+
+    current = path.mount_path if strip_prefix else path.virtual
+    results = [current] if include_root else []
+    if not resolved.is_dir or (maxdepth is not None and depth >= maxdepth):
+        return results
+
+    try:
+        children = await readdir(accessor, path, index)
+    except (FileNotFoundError, NotADirectoryError):
+        if ignore_missing:
+            return results
+        raise
+
+    for child in children:
+        child_path = PathSpec.from_str_path(
+            child, rekey(path.virtual, path.resource_path, child))
+        results.extend(await walk(accessor,
+                                  child_path,
+                                  index,
+                                  include_root=True,
+                                  maxdepth=maxdepth,
+                                  strip_prefix=strip_prefix,
+                                  ignore_missing=ignore_missing,
+                                  depth=depth + 1))
+    return results

python/mirage/core/discord/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/mirage/core/discord/_client.py

@@ -0,0 +1,83 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+
+import aiohttp
+
+from mirage.resource.discord.config import DiscordConfig
+from mirage.resource.secrets import reveal_secret
+
+DISCORD_API = "https://discord.com/api/v10"
+MAX_RETRIES = 3
+
+
+def discord_headers(config: DiscordConfig) -> dict[str, str]:
+    return {"Authorization": f"Bot {reveal_secret(config.token)}"}
+
+
+async def discord_get(
+    config: DiscordConfig,
+    endpoint: str,
+    params: dict | None = None,
+) -> dict | list:
+    url = f"{DISCORD_API}{endpoint}"
+    headers = discord_headers(config)
+    for attempt in range(MAX_RETRIES):
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=headers,
+                                   params=params) as resp:
+                if resp.status == 429:
+                    data = await resp.json()
+                    retry = data.get("retry_after", 1)
+                    if attempt < MAX_RETRIES - 1:
+                        await asyncio.sleep(retry)
+                        continue
+                    raise RuntimeError(
+                        f"Rate limited after {MAX_RETRIES} retries")
+                resp.raise_for_status()
+                return await resp.json()
+    return []
+
+
+async def discord_post(
+    config: DiscordConfig,
+    endpoint: str,
+    body: dict | None = None,
+) -> dict:
+    url = f"{DISCORD_API}{endpoint}"
+    headers = discord_headers(config)
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, json=body or {}) as resp:
+            if resp.status == 429:
+                data = await resp.json()
+                retry = data.get("retry_after", 1)
+                raise RuntimeError(f"Rate limited, retry after {retry}s")
+            resp.raise_for_status()
+            return await resp.json()
+
+
+async def discord_put(
+    config: DiscordConfig,
+    endpoint: str,
+) -> None:
+    url = f"{DISCORD_API}{endpoint}"
+    headers = discord_headers(config)
+    async with aiohttp.ClientSession() as session:
+        async with session.put(url, headers=headers) as resp:
+            if resp.status == 429:
+                data = await resp.json()
+                retry = data.get("retry_after", 1)
+                raise RuntimeError(f"Rate limited, retry after {retry}s")
+            resp.raise_for_status()

python/mirage/core/discord/channels.py

@@ -0,0 +1,62 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import AsyncIterator
+
+from mirage.core.discord._client import discord_get
+from mirage.resource.discord.config import DiscordConfig
+
+TEXT_CHANNEL_TYPES = (0, 5, 15)
+
+
+async def list_channels_stream(
+    config: DiscordConfig,
+    guild_id: str,
+) -> AsyncIterator[list[dict]]:
+    """List text channels in a guild as a single-page stream.
+
+    Discord returns all guild channels in one response (no pagination
+    cursor); the stream interface is provided for API parity with other
+    fetchers.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        guild_id (str): guild ID.
+
+    Yields:
+        list[dict]: filtered text-channel dicts.
+    """
+    raw = await discord_get(config, f"/guilds/{guild_id}/channels")
+    if not isinstance(raw, list):
+        return
+    yield [c for c in raw if c.get("type") in TEXT_CHANNEL_TYPES]
+
+
+async def list_channels(
+    config: DiscordConfig,
+    guild_id: str,
+) -> list[dict]:
+    """List text channels in a guild.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        guild_id (str): guild ID.
+
+    Returns:
+        list[dict]: channel dicts (text channels only).
+    """
+    out: list[dict] = []
+    async for page in list_channels_stream(config, guild_id):
+        out.extend(page)
+    return out

python/mirage/core/discord/entry.py

@@ -0,0 +1,104 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from datetime import datetime, timezone
+from enum import Enum
+
+from mirage.cache.index.config import IndexEntry
+from mirage.core.discord.history import DISCORD_EPOCH
+from mirage.core.timeutil import epoch_to_iso
+from mirage.utils.naming import make_id_name
+
+
+class DiscordResourceType(str, Enum):
+    GUILD = "discord/guild"
+    CHANNEL = "discord/channel"
+    MEMBER = "discord/member"
+    HISTORY = "discord/history"
+
+
+def guild_dirname(g: dict) -> str:
+    return make_id_name(g.get("name", ""), g["id"], path_safe=True)
+
+
+def channel_dirname(c: dict) -> str:
+    return make_id_name(c.get("name", ""), c["id"], path_safe=True)
+
+
+def member_filename(m: dict) -> str:
+    user = m.get("user", {})
+    return (
+        f"{make_id_name(user.get('username', ''), user['id'], path_safe=True)}"
+        ".json")
+
+
+def snowflake_to_date(snowflake: str) -> str:
+    """Convert a Discord snowflake to a UTC YYYY-MM-DD date string."""
+    ms = (int(snowflake) >> 22) + DISCORD_EPOCH
+    return datetime.fromtimestamp(ms / 1000,
+                                  tz=timezone.utc).strftime("%Y-%m-%d")
+
+
+def snowflake_to_iso(snowflake: str) -> str | None:
+    """Convert a Discord snowflake to a UTC ISO-8601 timestamp (second
+    precision, matching the TypeScript converter).
+
+    Args:
+        snowflake (str): the Discord snowflake id (empty/invalid -> None).
+    """
+    if not snowflake:
+        return None
+    try:
+        ms = (int(snowflake) >> 22) + DISCORD_EPOCH
+    except (TypeError, ValueError):
+        return None
+    return epoch_to_iso(ms // 1000)
+
+
+def guild_entry(g: dict) -> IndexEntry:
+    return IndexEntry(
+        id=g["id"],
+        name=g.get("name", ""),
+        resource_type=DiscordResourceType.GUILD,
+        vfs_name=guild_dirname(g),
+    )
+
+
+def channel_entry(c: dict) -> IndexEntry:
+    return IndexEntry(
+        id=c["id"],
+        name=c.get("name", ""),
+        resource_type=DiscordResourceType.CHANNEL,
+        vfs_name=channel_dirname(c),
+        remote_time=c.get("last_message_id", "") or "",
+    )
+
+
+def member_entry(m: dict) -> IndexEntry:
+    user = m.get("user", {})
+    return IndexEntry(
+        id=user.get("id", ""),
+        name=user.get("username", ""),
+        resource_type=DiscordResourceType.MEMBER,
+        vfs_name=member_filename(m),
+    )
+
+
+def history_entry(channel_key: str, date: str) -> IndexEntry:
+    return IndexEntry(
+        id=f"{channel_key}:{date}",
+        name=date,
+        resource_type=DiscordResourceType.HISTORY,
+        vfs_name=f"{date}.jsonl",
+    )

python/mirage/core/discord/files.py

@@ -0,0 +1,55 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import aiohttp
+
+from mirage.utils.sanitize import path_safe_name
+
+
+def file_blob_name(att: dict) -> str:
+    """Construct a stable VFS filename for a Discord attachment.
+
+    Args:
+        att (dict): Discord attachment dict (with id, filename fields).
+
+    Returns:
+        str: VFS filename of shape ``<stem>__<att-id>.<ext>``. The stem
+        keeps the original spelling, only ``/`` is replaced.
+    """
+    raw_name = att.get("filename") or att.get("title") or "file"
+    aid = str(att.get("id", ""))
+    if "." in raw_name:
+        stem, _, ext = raw_name.rpartition(".")
+        return f"{path_safe_name(stem)}__{aid}.{ext}"
+    return f"{path_safe_name(raw_name)}__{aid}"
+
+
+async def download_file(url: str) -> bytes:
+    """Download a Discord-hosted file blob.
+
+    Discord CDN URLs (``cdn.discordapp.com`` for ``url``,
+    ``media.discordapp.net`` for ``proxy_url``) are served without
+    authentication so no token is needed.
+
+    Args:
+        url (str): Discord attachment URL (typically ``url`` from the
+            attachment object).
+
+    Returns:
+        bytes: raw file content.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url) as resp:
+            resp.raise_for_status()
+            return await resp.read()

python/mirage/core/discord/formatters.py

@@ -0,0 +1,43 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+
+def format_grep_results(
+    messages: list[dict],
+    prefix: str,
+    guild_dirname: str,
+    channel_names: dict[str, str] | None = None,
+) -> list[str]:
+    """Format guild-search hits as grep-style lines.
+
+    Args:
+        messages (list[dict]): Discord message dicts from search_guild.
+        prefix (str): mount prefix, e.g. ``"/discord"``.
+        guild_dirname (str): vfs-safe guild dir name.
+        channel_names (dict[str, str] | None): channel_id → vfs name.
+
+    Returns:
+        list[str]: grep-style lines, one per matched message.
+    """
+    names = channel_names or {}
+    lines: list[str] = []
+    for msg in messages:
+        ts = msg.get("timestamp", "")[:10]
+        ch_id = msg.get("channel_id", "")
+        ch_name = names.get(ch_id, ch_id)
+        author = msg.get("author", {}).get("username", "?")
+        content = msg.get("content", "").replace("\n", " ")
+        lines.append(f"{prefix}/{guild_dirname}/channels/{ch_name}/"
+                     f"{ts}/chat.jsonl:[{author}] {content}")
+    return lines

python/mirage/core/discord/glob.py

@@ -0,0 +1,29 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.discord import DiscordAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.constants import SCOPE_ERROR
+from mirage.core.discord.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: DiscordAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore | None = None,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/discord/guilds.py

@@ -0,0 +1,59 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import AsyncIterator
+
+from mirage.core.discord.paginate import after_id_pages
+from mirage.resource.discord.config import DiscordConfig
+
+
+def list_guilds_stream(
+    config: DiscordConfig,
+    page_size: int = 200,
+) -> AsyncIterator[list[dict]]:
+    """Page-streaming variant of list_guilds.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        page_size (int): per-page limit (Discord caps at 200).
+
+    Yields:
+        list[dict]: guild dicts per page.
+    """
+    return after_id_pages(
+        config,
+        "/users/@me/guilds",
+        base_params={},
+        last_id_fn=lambda g: g["id"],
+        page_size=page_size,
+    )
+
+
+async def list_guilds(
+    config: DiscordConfig,
+    page_size: int = 200,
+) -> list[dict]:
+    """List all guilds the bot is in (paginated).
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        page_size (int): per-page limit.
+
+    Returns:
+        list[dict]: guild dicts with id, name.
+    """
+    out: list[dict] = []
+    async for page in list_guilds_stream(config, page_size):
+        out.extend(page)
+    return out

python/mirage/core/discord/history.py

@@ -0,0 +1,125 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import json
+from collections.abc import AsyncIterator
+from datetime import datetime, timezone
+
+from mirage.core.discord.paginate import after_id_pages
+from mirage.resource.discord.config import DiscordConfig
+
+DISCORD_EPOCH = 1420070400000
+
+
+def date_to_snowflake(date_str: str, end: bool = False) -> str:
+    """Convert a YYYY-MM-DD date to a Discord snowflake bound.
+
+    Args:
+        date_str (str): YYYY-MM-DD date.
+        end (bool): when True, returns the snowflake for 23:59:59 UTC.
+
+    Returns:
+        str: snowflake id usable as ``after``/``before`` parameter.
+    """
+    dt = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+    if end:
+        dt = dt.replace(hour=23, minute=59, second=59)
+    ms = int(dt.timestamp() * 1000) - DISCORD_EPOCH
+    return str(ms << 22)
+
+
+async def stream_messages_for_day(
+    config: DiscordConfig,
+    channel_id: str,
+    date_str: str,
+    page_size: int = 100,
+) -> AsyncIterator[list[dict]]:
+    """Stream message pages for a channel-day.
+
+    Walks ``/channels/<id>/messages?after=<snowflake>&limit=N``
+    forward through the day, stopping when messages exceed the
+    end-of-day snowflake.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        channel_id (str): channel ID.
+        date_str (str): YYYY-MM-DD.
+        page_size (int): per-page limit (Discord caps at 100).
+
+    Yields:
+        list[dict]: message dicts, filtered to within the date.
+    """
+    after = date_to_snowflake(date_str)
+    before_int = int(date_to_snowflake(date_str, end=True))
+    async for page in after_id_pages(
+            config,
+            f"/channels/{channel_id}/messages",
+            base_params={},
+            last_id_fn=lambda m: m["id"],
+            page_size=page_size,
+            start_after=after,
+    ):
+        in_range = [m for m in page if int(m["id"]) <= before_int]
+        if in_range:
+            yield in_range
+        if any(int(m["id"]) > before_int for m in page):
+            return
+
+
+async def list_messages_for_day(
+    config: DiscordConfig,
+    channel_id: str,
+    date_str: str,
+    page_size: int = 100,
+) -> list[dict]:
+    """List all messages for a channel-day (eager).
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        channel_id (str): channel ID.
+        date_str (str): YYYY-MM-DD.
+        page_size (int): per-page limit.
+
+    Returns:
+        list[dict]: messages within the date, sorted oldest-first.
+    """
+    out: list[dict] = []
+    async for page in stream_messages_for_day(config, channel_id, date_str,
+                                              page_size):
+        out.extend(page)
+    out.sort(key=lambda m: int(m["id"]))
+    return out
+
+
+async def get_history_jsonl(
+    config: DiscordConfig,
+    channel_id: str,
+    date_str: str,
+) -> bytes:
+    """Fetch channel messages for a date as JSONL.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        channel_id (str): channel ID.
+        date_str (str): date in YYYY-MM-DD format.
+
+    Returns:
+        bytes: JSONL-encoded messages.
+    """
+    messages = await list_messages_for_day(config, channel_id, date_str)
+    lines = [
+        json.dumps(m, ensure_ascii=False, separators=(",", ":"))
+        for m in messages
+    ]
+    return ("\n".join(lines) + "\n").encode() if lines else b""

python/mirage/core/discord/members.py

@@ -0,0 +1,99 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import AsyncIterator
+
+from mirage.core.discord._client import discord_get
+from mirage.core.discord.paginate import after_id_pages
+from mirage.resource.discord.config import DiscordConfig
+
+
+def _member_user_id(m: dict) -> str:
+    return m.get("user", {}).get("id", "")
+
+
+def list_members_stream(
+    config: DiscordConfig,
+    guild_id: str,
+    page_size: int = 1000,
+) -> AsyncIterator[list[dict]]:
+    """Stream guild members across pages.
+
+    Walks ``/guilds/<id>/members?after=<user_id>&limit=N`` until the
+    API returns a partial page.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        guild_id (str): guild ID.
+        page_size (int): per-page limit (Discord caps at 1000).
+
+    Yields:
+        list[dict]: member dicts per page.
+    """
+    return after_id_pages(
+        config,
+        f"/guilds/{guild_id}/members",
+        base_params={},
+        last_id_fn=_member_user_id,
+        page_size=page_size,
+    )
+
+
+async def list_members(
+    config: DiscordConfig,
+    guild_id: str,
+    page_size: int = 1000,
+) -> list[dict]:
+    """List all guild members (paginated).
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        guild_id (str): guild ID.
+        page_size (int): per-page limit.
+
+    Returns:
+        list[dict]: member dicts.
+    """
+    out: list[dict] = []
+    async for page in list_members_stream(config, guild_id, page_size):
+        out.extend(page)
+    return out
+
+
+async def search_members(
+    config: DiscordConfig,
+    guild_id: str,
+    query: str,
+    limit: int = 100,
+) -> list[dict]:
+    """Search guild members by name.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        guild_id (str): guild ID.
+        query (str): search query.
+        limit (int): max results.
+
+    Returns:
+        list[dict]: matching members.
+    """
+    result = await discord_get(
+        config,
+        f"/guilds/{guild_id}/members/search",
+        params={
+            "query": query,
+            "limit": limit
+        },
+    )
+    return result if isinstance(result, list) else []

python/mirage/core/discord/paginate.py

@@ -0,0 +1,121 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import AsyncIterator, Callable
+from typing import Any
+
+from mirage.core.discord._client import discord_get
+from mirage.resource.discord.config import DiscordConfig
+
+
+async def after_id_pages(
+    config: DiscordConfig,
+    endpoint: str,
+    base_params: dict,
+    last_id_fn: Callable[[dict], str],
+    page_size: int = 100,
+    start_after: str = "0",
+) -> AsyncIterator[list[dict]]:
+    """Walk an after-id paginated Discord endpoint.
+
+    Used for endpoints that return a flat list and accept `after=<id>` +
+    `limit=<n>` for pagination (history, members, guilds).
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        endpoint (str): Discord API path, e.g. "/channels/X/messages".
+        base_params (dict): per-request params; "after" + "limit" are set
+            here.
+        last_id_fn (Callable[[dict], str]): extract the cursor id from
+            the last item of a page (e.g. ``lambda m: m["id"]`` for
+            messages, ``lambda m: m["user"]["id"]`` for members).
+        page_size (int): per-page limit (Discord caps vary by endpoint).
+        start_after (str): initial "after" cursor.
+
+    Yields:
+        list[dict]: items in each page. Generator returns when the API
+        returns an empty page or a partial page (signalling end).
+    """
+    last = start_after
+    while True:
+        params = dict(base_params)
+        params["after"] = last
+        params["limit"] = page_size
+        data = await discord_get(config, endpoint, params=params)
+        if not isinstance(data, list) or not data:
+            return
+        yield data
+        if len(data) < page_size:
+            return
+        last = last_id_fn(data[-1])
+
+
+def _get_nested(d: dict, path: tuple[str, ...]) -> Any:
+    cur: Any = d
+    for k in path:
+        if not isinstance(cur, dict):
+            return None
+        cur = cur.get(k)
+    return cur
+
+
+async def offset_pages(
+    config: DiscordConfig,
+    endpoint: str,
+    base_params: dict,
+    items_path: tuple[str, ...],
+    total_key: str = "total_results",
+    page_size: int = 25,
+    start_offset: int = 0,
+    max_pages: int | None = None,
+) -> AsyncIterator[list[Any]]:
+    """Walk an offset-paginated Discord endpoint (search).
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        endpoint (str): Discord API path, e.g.
+            "/guilds/X/messages/search".
+        base_params (dict): per-request params; "offset" is set here.
+        items_path (tuple[str, ...]): nested path to items list in the
+            response, e.g. ``("messages",)`` for search.
+        total_key (str): top-level key holding the total result count.
+        page_size (int): per-page count (Discord search returns 25).
+        start_offset (int): initial offset.
+        max_pages (int | None): cap on pages fetched; None = unbounded.
+
+    Yields:
+        list[Any]: items from each page (search returns context arrays;
+        the caller flattens).
+    """
+    offset = start_offset
+    total: int | None = None
+    fetched = 0
+    while True:
+        params = dict(base_params)
+        params["offset"] = offset
+        data = await discord_get(config, endpoint, params=params)
+        if not isinstance(data, dict):
+            return
+        items = _get_nested(data, items_path) or []
+        if not items:
+            return
+        yield items
+        fetched += 1
+        if total is None:
+            total = data.get(total_key, 0)
+        offset += page_size
+        if total is not None and offset >= total:
+            return
+        if max_pages is not None and fetched >= max_pages:
+            return

python/mirage/core/discord/post.py

@@ -0,0 +1,43 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.core.discord._client import discord_post
+from mirage.resource.discord.config import DiscordConfig
+
+
+async def send_message(
+    config: DiscordConfig,
+    channel_id: str,
+    text: str,
+    message_reference_id: str | None = None,
+) -> dict:
+    """Send a message to a channel.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        channel_id (str): channel ID.
+        text (str): message content.
+        message_reference_id (str | None): reply to message.
+
+    Returns:
+        dict: API response.
+    """
+    body: dict = {"content": text}
+    if message_reference_id:
+        body["message_reference"] = {"message_id": message_reference_id}
+    return await discord_post(
+        config,
+        f"/channels/{channel_id}/messages",
+        body,
+    )

python/mirage/core/discord/react.py

@@ -0,0 +1,40 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from urllib.parse import quote
+
+from mirage.core.discord._client import discord_put
+from mirage.resource.discord.config import DiscordConfig
+
+
+async def add_reaction(
+    config: DiscordConfig,
+    channel_id: str,
+    message_id: str,
+    emoji: str,
+) -> None:
+    """Add a reaction to a message.
+
+    Args:
+        config (DiscordConfig): Discord credentials.
+        channel_id (str): channel ID.
+        message_id (str): message ID.
+        emoji (str): emoji name or unicode.
+    """
+    encoded = quote(emoji, safe="")
+    await discord_put(
+        config,
+        f"/channels/{channel_id}/messages"
+        f"/{message_id}/reactions/{encoded}/@me",
+    )

python/mirage/core/discord/read.py

@@ -0,0 +1,110 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import json
+
+from mirage.accessor.discord import DiscordAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.discord.files import download_file
+from mirage.core.discord.history import get_history_jsonl
+from mirage.core.discord.members import list_members
+from mirage.core.discord.readdir import readdir as _readdir
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+
+async def _ensure_channel(
+    index: IndexCacheStore,
+    prefix: str,
+    ch_key: str,
+    virtual: str,
+):
+    ch_virtual = prefix + "/" + ch_key
+    lookup = await index.get(ch_virtual)
+    if lookup.entry is None:
+        raise enoent(virtual)
+    return lookup
+
+
+async def read(
+    accessor: DiscordAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> bytes:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual if isinstance(path, PathSpec) else path
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    key = path.resource_path
+    parts = key.split("/")
+
+    # <guild>/channels/<ch>/<date>/chat.jsonl
+    if (len(parts) == 5 and parts[1] == "channels"
+            and parts[4] == "chat.jsonl"):
+        if index is None:
+            raise enoent(virtual)
+        ch_key = f"{parts[0]}/{parts[1]}/{parts[2]}"
+        ch_lookup = await _ensure_channel(index, prefix, ch_key, virtual)
+        return await get_history_jsonl(accessor.config, ch_lookup.entry.id,
+                                       parts[3])
+
+    # <guild>/channels/<ch>/<date>/files/<blob>
+    if (len(parts) == 6 and parts[1] == "channels" and parts[4] == "files"):
+        if index is None:
+            raise enoent(virtual)
+        virtual_key = prefix + "/" + key
+        lookup = await index.get(virtual_key)
+        if lookup.entry is None:
+            # Hydrate via date dir readdir, which triggers _fetch_day
+            date_key = "/".join(parts[:4])
+            date_spec = PathSpec(
+                virtual=prefix + "/" + date_key,
+                directory=prefix + "/" + date_key,
+                resource_path=mount_key(prefix + "/" + date_key, prefix),
+            )
+            await _readdir(accessor, date_spec, index)
+            lookup = await index.get(virtual_key)
+        if lookup.entry is None:
+            raise enoent(virtual)
+        url = (lookup.entry.extra
+               or {}).get("url") or (lookup.entry.extra
+                                     or {}).get("proxy_url") or ""
+        if not url:
+            raise enoent(virtual)
+        return await download_file(url)
+
+    # <guild>/members/<user>.json
+    if len(parts) == 3 and parts[1] == "members":
+        if index is None:
+            raise enoent(virtual)
+        virtual_key = prefix + "/" + key
+        entry_lookup = await index.get(virtual_key)
+        if entry_lookup.entry is None:
+            raise enoent(virtual)
+        guild_virtual = prefix + "/" + parts[0]
+        guild_lookup = await index.get(guild_virtual)
+        if guild_lookup.entry is None:
+            raise enoent(virtual)
+        members = await list_members(accessor.config, guild_lookup.entry.id)
+        for m in members:
+            user = m.get("user", {})
+            if user.get("id") == entry_lookup.entry.id:
+                return json.dumps(m, ensure_ascii=False,
+                                  separators=(",", ":")).encode()
+        raise enoent(virtual)
+
+    raise enoent(virtual)

python/mirage/core/discord/readdir.py

@@ -0,0 +1,397 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+import aiohttp
+
+from mirage.accessor.discord import DiscordAccessor
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.discord.channels import list_channels
+from mirage.core.discord.entry import (DiscordResourceType, channel_entry,
+                                       guild_entry, history_entry,
+                                       member_entry, snowflake_to_date)
+from mirage.core.discord.files import file_blob_name
+from mirage.core.discord.guilds import list_guilds
+from mirage.core.discord.history import list_messages_for_day
+from mirage.core.discord.members import list_members
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+logger = logging.getLogger(__name__)
+
+SOFT_HTTP_STATUSES = frozenset((403, 404, 429))
+
+
+def _is_soft_error(exc: Exception) -> bool:
+    return (isinstance(exc, aiohttp.ClientResponseError)
+            and exc.status in SOFT_HTTP_STATUSES)
+
+
+def _date_range(end_date: str, days: int = 30) -> list[str]:
+    end = datetime.strptime(end_date, "%Y-%m-%d").date()
+    return [(end - timedelta(days=i)).isoformat()
+            for i in range(days - 1, -1, -1)]
+
+
+def _normalize_path(path: PathSpec | str) -> tuple[str, str, str]:
+    """Reduce input to (prefix, key, virtual_key)."""
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    raw = path.directory if path.pattern else path.virtual
+    if prefix and raw.startswith(prefix):
+        raw = raw[len(prefix):] or "/"
+    key = raw.strip("/")
+    virtual_key = prefix + "/" + key if key else prefix or "/"
+    return prefix, key, virtual_key
+
+
+async def _readdir_root(
+    accessor: DiscordAccessor,
+    prefix: str,
+    virtual_key: str,
+    index: IndexCacheStore | None,
+) -> list[str]:
+    if index is not None:
+        listing = await index.list_dir(virtual_key)
+        if listing.entries is not None:
+            return listing.entries
+    guilds = await list_guilds(accessor.config)
+    entries = []
+    names = []
+    for g in guilds:
+        entry = guild_entry(g)
+        entries.append((entry.vfs_name, entry))
+        names.append(f"{prefix}/{entry.vfs_name}")
+    if index is not None:
+        await index.set_dir(virtual_key, entries)
+    return names
+
+
+async def _ensure_guild_id(
+    accessor: DiscordAccessor,
+    prefix: str,
+    guild_part: str,
+    index: IndexCacheStore | None,
+    raw_path: str,
+) -> str:
+    if index is None:
+        raise enoent(raw_path)
+    guild_virtual_key = prefix + "/" + guild_part
+    lookup = await index.get(guild_virtual_key)
+    if lookup.entry is None:
+        await _readdir_root(accessor, prefix, prefix or "/", index)
+        lookup = await index.get(guild_virtual_key)
+    if lookup.entry is None:
+        raise enoent(raw_path)
+    return lookup.entry.id
+
+
+async def _readdir_guild_top(
+    prefix: str,
+    key: str,
+) -> list[str]:
+    return [f"{prefix}/{key}/channels", f"{prefix}/{key}/members"]
+
+
+async def _readdir_channels(
+    accessor: DiscordAccessor,
+    prefix: str,
+    key: str,
+    virtual_key: str,
+    parts: list[str],
+    index: IndexCacheStore | None,
+    raw_path: str,
+) -> list[str]:
+    if index is not None:
+        listing = await index.list_dir(virtual_key)
+        if listing.entries is not None:
+            return listing.entries
+    guild_id = await _ensure_guild_id(accessor, prefix, parts[0], index,
+                                      raw_path)
+    channels = await list_channels(accessor.config, guild_id)
+    entries = []
+    names = []
+    for c in channels:
+        entry = channel_entry(c)
+        entries.append((entry.vfs_name, entry))
+        names.append(f"{prefix}/{key}/{entry.vfs_name}")
+    if index is not None:
+        await index.set_dir(virtual_key, entries)
+    return names
+
+
+async def _readdir_members(
+    accessor: DiscordAccessor,
+    prefix: str,
+    key: str,
+    virtual_key: str,
+    parts: list[str],
+    index: IndexCacheStore | None,
+    raw_path: str,
+) -> list[str]:
+    if index is not None:
+        listing = await index.list_dir(virtual_key)
+        if listing.entries is not None:
+            return listing.entries
+    guild_id = await _ensure_guild_id(accessor, prefix, parts[0], index,
+                                      raw_path)
+    members = await list_members(accessor.config, guild_id)
+    entries = []
+    names = []
+    for m in members:
+        entry = member_entry(m)
+        entries.append((entry.vfs_name, entry))
+        names.append(f"{prefix}/{key}/{entry.vfs_name}")
+    if index is not None:
+        await index.set_dir(virtual_key, entries)
+    return names
+
+
+async def _ensure_channel_lookup(
+    accessor: DiscordAccessor,
+    prefix: str,
+    parts: list[str],
+    index: IndexCacheStore,
+    raw_path: str,
+):
+    channel_vk = f"{prefix}/{'/'.join(parts[:3])}"
+    lookup = await index.get(channel_vk)
+    if lookup.entry is None:
+        await _readdir_channels(accessor, prefix, "/".join(parts[:2]),
+                                f"{prefix}/{'/'.join(parts[:2])}", parts[:2],
+                                index, raw_path)
+        lookup = await index.get(channel_vk)
+    if lookup.entry is None:
+        raise enoent(raw_path)
+    return lookup
+
+
+async def _readdir_channel_dates(
+    accessor: DiscordAccessor,
+    prefix: str,
+    key: str,
+    virtual_key: str,
+    parts: list[str],
+    index: IndexCacheStore | None,
+    raw_path: str,
+) -> list[str]:
+    if index is None:
+        last_msg_id = ""
+    else:
+        listing = await index.list_dir(virtual_key)
+        if listing.entries is not None:
+            return listing.entries
+        lookup = await _ensure_channel_lookup(accessor, prefix, parts, index,
+                                              raw_path)
+        last_msg_id = lookup.entry.remote_time
+    if last_msg_id:
+        end_date = snowflake_to_date(last_msg_id)
+    else:
+        end_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
+    dates = _date_range(end_date)
+    entries = []
+    names = []
+    for d in dates:
+        entry = history_entry(key, d)
+        entry.resource_type = DiscordResourceType.HISTORY
+        entry.vfs_name = d
+        entries.append((d, entry))
+        names.append(f"{prefix}/{key}/{d}")
+    if index is not None:
+        await index.set_dir(virtual_key, entries)
+    return names
+
+
+async def _fetch_day(
+    accessor: DiscordAccessor,
+    channel_id: str,
+    date_str: str,
+    date_vkey: str,
+    index: IndexCacheStore,
+) -> None:
+    """Walk the day's history once, populate date dir and files dir
+    entries in the index. Tolerates soft HTTP errors (403/404/429) by
+    sealing an empty date dir.
+    """
+    try:
+        messages = await list_messages_for_day(accessor.config, channel_id,
+                                               date_str)
+    except aiohttp.ClientResponseError as e:
+        if _is_soft_error(e):
+            logger.debug("discord: history denied for %s/%s (%d); empty day",
+                         channel_id, date_str, e.status)
+            await index.set_dir(date_vkey, [])
+            return
+        raise
+    chat_entry = IndexEntry(
+        id=f"{channel_id}:{date_str}:chat",
+        name="chat.jsonl",
+        resource_type="discord/chat_jsonl",
+        vfs_name="chat.jsonl",
+    )
+    files_entry = IndexEntry(
+        id=f"{channel_id}:{date_str}:files",
+        name="files",
+        resource_type="discord/files_dir",
+        vfs_name="files",
+    )
+    await index.set_dir(date_vkey, [
+        ("chat.jsonl", chat_entry),
+        ("files", files_entry),
+    ])
+    file_entries: list[tuple[str, IndexEntry]] = []
+    for msg in messages:
+        for att in msg.get("attachments") or []:
+            if not att.get("id"):
+                continue
+            blob_name = file_blob_name(att)
+            file_entries.append((blob_name,
+                                 IndexEntry(
+                                     id=str(att["id"]),
+                                     name=att.get("filename") or "",
+                                     resource_type="discord/file",
+                                     vfs_name=blob_name,
+                                     size=att.get("size"),
+                                     extra={
+                                         "url":
+                                         att.get("url", ""),
+                                         "proxy_url":
+                                         att.get("proxy_url", ""),
+                                         "content_type":
+                                         att.get("content_type", ""),
+                                         "message_id":
+                                         msg.get("id", ""),
+                                         "author":
+                                         msg.get("author",
+                                                 {}).get("username", ""),
+                                         "channel_id":
+                                         channel_id,
+                                         "date":
+                                         date_str,
+                                     },
+                                 )))
+    await index.set_dir(f"{date_vkey}/files", file_entries)
+
+
+async def _readdir_date_contents(
+    accessor: DiscordAccessor,
+    prefix: str,
+    key: str,
+    virtual_key: str,
+    parts: list[str],
+    index: IndexCacheStore | None,
+    raw_path: str,
+) -> list[str]:
+    if index is None:
+        raise enoent(raw_path)
+    cached = await index.list_dir(virtual_key)
+    if cached.entries is not None:
+        return cached.entries
+    lookup = await _ensure_channel_lookup(accessor, prefix, parts, index,
+                                          raw_path)
+    await _fetch_day(accessor, lookup.entry.id, parts[3], virtual_key, index)
+    cached = await index.list_dir(virtual_key)
+    if cached.entries is None:
+        raise enoent(raw_path)
+    return cached.entries
+
+
+async def _readdir_files_dir(
+    accessor: DiscordAccessor,
+    prefix: str,
+    key: str,
+    virtual_key: str,
+    parts: list[str],
+    index: IndexCacheStore | None,
+    raw_path: str,
+) -> list[str]:
+    if index is None:
+        raise enoent(raw_path)
+    cached = await index.list_dir(virtual_key)
+    if cached.entries is not None:
+        return cached.entries
+    # Date dir lookup triggers _fetch_day which populates the files dir
+    date_key = "/".join(parts[:4])
+    date_vk = f"{prefix}/{date_key}"
+    date_spec = PathSpec(virtual=date_vk,
+                         directory=date_vk,
+                         resource_path=mount_key(date_vk, prefix))
+    await readdir(accessor, date_spec, index)
+    cached = await index.list_dir(virtual_key)
+    if cached.entries is None:
+        raise enoent(raw_path)
+    return cached.entries
+
+
+async def readdir(
+    accessor: DiscordAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> list[str]:
+    """List directory contents.
+
+    Args:
+        accessor (DiscordAccessor): discord accessor.
+        path (PathSpec | str): resource-relative path.
+        index (IndexCacheStore | None): index cache.
+    """
+    prefix, key, virtual_key = _normalize_path(path)
+    raw_path = path.virtual if isinstance(path, PathSpec) else path
+
+    if not key:
+        return await _readdir_root(accessor, prefix, virtual_key, index)
+
+    parts = key.split("/")
+
+    if len(parts) == 1:
+        if index is not None:
+            lookup = await index.get(virtual_key)
+            if lookup.entry is None:
+                raise enoent(raw_path)
+        return await _readdir_guild_top(prefix, key)
+
+    if len(parts) == 2 and parts[1] == "channels":
+        return await _readdir_channels(accessor, prefix, key, virtual_key,
+                                       parts, index, raw_path)
+
+    if len(parts) == 2 and parts[1] == "members":
+        return await _readdir_members(accessor, prefix, key, virtual_key,
+                                      parts, index, raw_path)
+
+    if len(parts) == 3 and parts[1] == "channels":
+        return await _readdir_channel_dates(accessor, prefix, key, virtual_key,
+                                            parts, index, raw_path)
+
+    if len(parts) == 4 and parts[1] == "channels":
+        return await _readdir_date_contents(accessor, prefix, key, virtual_key,
+                                            parts, index, raw_path)
+
+    if (len(parts) == 5 and parts[1] == "channels" and parts[4] == "files"):
+        return await _readdir_files_dir(accessor, prefix, key, virtual_key,
+                                        parts, index, raw_path)
+
+    return []
+
+
+def is_dir_name(child: str) -> bool:
+    # Entries are recognized by extension, so classification never needs
+    # the stat fallback.
+    name = child.rsplit("/", 1)[-1]
+    return not (name.endswith(".json") or name.endswith(".jsonl"))

python/mirage/core/discord/scope.py

@@ -0,0 +1,239 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import re
+from dataclasses import dataclass
+
+from mirage.cache.index import IndexCacheStore
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_prefix_of
+
+_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+
+
+@dataclass
+class DiscordScope:
+    """Resolved scope for a discord path.
+
+    Attributes:
+        level (str): one of ``root``, ``guild``, ``channel``, ``date``,
+            ``messages``, ``files``, ``file_blob``, ``member``.
+        guild_id (str | None): guild snowflake.
+        channel_id (str | None): channel snowflake.
+        date_str (str | None): ``YYYY-MM-DD`` for date-level and below.
+        resource_path (str): resource-relative key (prefix stripped).
+    """
+
+    level: str
+    guild_id: str | None = None
+    channel_id: str | None = None
+    date_str: str | None = None
+    resource_path: str = "/"
+
+
+def _strip_prefix(raw: str, prefix: str) -> str:
+    stripped = raw.strip("/")
+    if not prefix:
+        return stripped
+    pfx = prefix.strip("/")
+    if stripped == pfx:
+        return ""
+    if stripped.startswith(pfx + "/"):
+        return stripped[len(pfx) + 1:]
+    return stripped
+
+
+async def detect_scope(
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> DiscordScope:
+    """Determine scope from a path.
+
+    Examples::
+
+        /                                              → root
+        /<guild>                                       → guild
+        /<guild>/channels                              → guild
+        /<guild>/members                               → guild
+        /<guild>/channels/<ch>                         → channel
+        /<guild>/members/<user>.json                   → member
+        /<guild>/channels/<ch>/<date>                  → date
+        /<guild>/channels/<ch>/<date>/chat.jsonl       → messages
+        /<guild>/channels/<ch>/<date>/files            → files
+        /<guild>/channels/<ch>/<date>/files/<blob>     → file_blob
+    """
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+
+    prefix = mount_prefix_of(path.virtual, path.resource_path) or ""
+
+    if path.pattern and path.pattern.endswith(".jsonl"):
+        dir_key = _strip_prefix(path.directory, prefix)
+        parts = dir_key.split("/") if dir_key else []
+        if len(parts) >= 3 and parts[1] == "channels":
+            guild_id, channel_id = await _resolve_ids(parts[0],
+                                                      "/".join(parts[:3]),
+                                                      index, prefix)
+            date_str = (parts[3] if len(parts) == 4
+                        and _DATE_RE.match(parts[3]) else None)
+            return DiscordScope(
+                level="messages" if date_str else "channel",
+                guild_id=guild_id,
+                channel_id=channel_id,
+                date_str=date_str,
+                resource_path=dir_key,
+            )
+
+    key = _strip_prefix(path.virtual, prefix)
+
+    if not key:
+        return DiscordScope(level="root", resource_path="/")
+
+    parts = key.split("/")
+
+    # /<guild>/channels/<ch>/<date>/files/<blob>
+    if (len(parts) == 6 and parts[1] == "channels" and _DATE_RE.match(parts[3])
+            and parts[4] == "files"):
+        guild_id, channel_id = await _resolve_ids(parts[0],
+                                                  "/".join(parts[:3]), index,
+                                                  prefix)
+        return DiscordScope(
+            level="file_blob",
+            guild_id=guild_id,
+            channel_id=channel_id,
+            date_str=parts[3],
+            resource_path=key,
+        )
+
+    # /<guild>/channels/<ch>/<date>/chat.jsonl or .../files
+    if (len(parts) == 5 and parts[1] == "channels"
+            and _DATE_RE.match(parts[3])):
+        guild_id, channel_id = await _resolve_ids(parts[0],
+                                                  "/".join(parts[:3]), index,
+                                                  prefix)
+        if parts[4] == "chat.jsonl":
+            return DiscordScope(
+                level="messages",
+                guild_id=guild_id,
+                channel_id=channel_id,
+                date_str=parts[3],
+                resource_path=key,
+            )
+        if parts[4] == "files":
+            return DiscordScope(
+                level="files",
+                guild_id=guild_id,
+                channel_id=channel_id,
+                date_str=parts[3],
+                resource_path=key,
+            )
+
+    # /<guild>/channels/<ch>/<date>
+    if (len(parts) == 4 and parts[1] == "channels"
+            and _DATE_RE.match(parts[3])):
+        guild_id, channel_id = await _resolve_ids(parts[0],
+                                                  "/".join(parts[:3]), index,
+                                                  prefix)
+        return DiscordScope(
+            level="date",
+            guild_id=guild_id,
+            channel_id=channel_id,
+            date_str=parts[3],
+            resource_path=key,
+        )
+
+    # /<guild>/channels/<ch>
+    if len(parts) == 3 and parts[1] == "channels":
+        guild_id, channel_id = await _resolve_ids(parts[0], key, index, prefix)
+        return DiscordScope(
+            level="channel",
+            guild_id=guild_id,
+            channel_id=channel_id,
+            resource_path=key,
+        )
+
+    # /<guild>/members/<user>.json
+    if len(parts) == 3 and parts[1] == "members":
+        guild_id = await _resolve_guild_id(parts[0], index, prefix)
+        return DiscordScope(
+            level="member",
+            guild_id=guild_id,
+            resource_path=key,
+        )
+
+    # /<guild>, /<guild>/channels, /<guild>/members
+    if len(parts) <= 2:
+        guild_id = await _resolve_guild_id(parts[0], index, prefix)
+        return DiscordScope(
+            level="guild",
+            guild_id=guild_id,
+            resource_path=key,
+        )
+
+    return DiscordScope(level="file_blob", resource_path=key)
+
+
+async def coalesce_scopes(
+    paths: list[PathSpec],
+    index: IndexCacheStore = None,
+) -> DiscordScope | None:
+    if not paths:
+        return None
+    scopes = [await detect_scope(p, index) for p in paths]
+    first = scopes[0]
+    if first.guild_id is None or first.channel_id is None:
+        return None
+    for s in scopes[1:]:
+        if (s.guild_id != first.guild_id or s.channel_id != first.channel_id):
+            return None
+    return DiscordScope(
+        level="channel",
+        guild_id=first.guild_id,
+        channel_id=first.channel_id,
+        resource_path=first.resource_path.rsplit("/", 1)[0]
+        if first.level == "messages" else first.resource_path,
+    )
+
+
+async def _resolve_guild_id(
+    guild_name: str,
+    index: IndexCacheStore | None,
+    prefix: str,
+) -> str | None:
+    if index is None:
+        return None
+    virtual_key = prefix + "/" + guild_name if prefix else "/" + guild_name
+    lookup = await index.get(virtual_key)
+    if lookup.entry is not None:
+        return lookup.entry.id
+    return None
+
+
+async def _resolve_ids(
+    guild_name: str,
+    channel_path: str,
+    index: IndexCacheStore | None,
+    prefix: str,
+) -> tuple[str | None, str | None]:
+    guild_id = await _resolve_guild_id(guild_name, index, prefix)
+    channel_id = None
+    if index is not None:
+        virtual_key = (prefix + "/" + channel_path if prefix else "/" +
+                       channel_path)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is not None:
+            channel_id = lookup.entry.id
+    return guild_id, channel_id

python/mirage/core/discord/search.py

@@ -0,0 +1,101 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import AsyncIterator
+
+from mirage.core.discord.paginate import offset_pages
+from mirage.resource.discord.config import DiscordConfig
+
+PAGE_SIZE = 25
+
+
+def _flatten_contexts(contexts: list) -> list[dict]:
+    """Pull the matched message from each search-context array.
+
+    Discord search responses are shaped as ``[[ctx_msg, ...], ...]``;
+    the matched message is the first entry of each context.
+    """
+    out: list[dict] = []
+    for context in contexts:
+        if isinstance(context, list) and context:
+            out.append(context[0])
+    return out
+
+
+async def search_guild_stream(
+    config: DiscordConfig,
+    guild_id: str,
+    query: str,
+    channel_id: str | None = None,
+    max_pages: int | None = None,
+) -> AsyncIterator[list[dict]]:
+    """Stream guild-search pages, one flattened batch per round-trip.
+
+    Args:
+        config (DiscordConfig): credentials.
+        guild_id (str): guild snowflake ID.
+        query (str): search text (content match).
+        channel_id (str | None): filter to specific channel.
+        max_pages (int | None): cap on pages fetched.
+
+    Yields:
+        list[dict]: matched message dicts per page (flattened from
+        Discord's context arrays).
+    """
+    base_params: dict[str, str | int] = {"content": query}
+    if channel_id:
+        base_params["channel_id"] = channel_id
+    async for raw in offset_pages(
+            config,
+            f"/guilds/{guild_id}/messages/search",
+            base_params=base_params,
+            items_path=("messages", ),
+            total_key="total_results",
+            page_size=PAGE_SIZE,
+            max_pages=max_pages,
+    ):
+        flat = _flatten_contexts(raw)
+        if flat:
+            yield flat
+
+
+async def search_guild(
+    config: DiscordConfig,
+    guild_id: str,
+    query: str,
+    channel_id: str | None = None,
+    limit: int = 100,
+) -> list[dict]:
+    """Search messages in a guild, optionally filtered to one channel.
+
+    Args:
+        config (DiscordConfig): credentials.
+        guild_id (str): guild snowflake ID.
+        query (str): search text (content match).
+        channel_id (str | None): filter to specific channel.
+        limit (int): max results to return.
+
+    Returns:
+        list[dict]: matching messages sorted oldest-first.
+    """
+    messages: list[dict] = []
+    async for page in search_guild_stream(config, guild_id, query, channel_id):
+        for msg in page:
+            messages.append(msg)
+            if len(messages) >= limit:
+                break
+        if len(messages) >= limit:
+            break
+    messages.sort(key=lambda m: int(m.get("id", 0)))
+    return messages[:limit]

python/mirage/core/discord/stat.py

@@ -0,0 +1,155 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import re
+
+from mirage.accessor.discord import DiscordAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.discord.entry import snowflake_to_iso
+from mirage.core.discord.readdir import readdir as _readdir
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.filetype import filetype_from_mimetype
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+VIRTUAL_DIRS = {"", "channels", "members"}
+_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+
+
+async def _populate_via_parent(
+    accessor: DiscordAccessor,
+    virtual_key: str,
+    prefix: str,
+    index: IndexCacheStore,
+) -> None:
+    parent_virtual = virtual_key.rsplit("/", 1)[0] or "/"
+    try:
+        await _readdir(
+            accessor,
+            PathSpec(virtual=parent_virtual,
+                     directory=parent_virtual,
+                     resource_path=mount_key(parent_virtual, prefix)),
+            index=index,
+        )
+    # best-effort cache populate; canonical ENOENT raised below
+    except Exception:
+        pass
+
+
+async def stat(
+    accessor: DiscordAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> FileStat:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    key = path.resource_path
+
+    if not key:
+        return FileStat(name="/", type=FileType.DIRECTORY)
+
+    parts = key.split("/")
+    virtual_key = prefix + "/" + key
+
+    if len(parts) == 1:
+        if index is None:
+            raise enoent(virtual)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is None:
+            await _populate_via_parent(accessor, virtual_key, prefix, index)
+            lookup = await index.get(virtual_key)
+            if lookup.entry is None:
+                raise enoent(virtual)
+        return FileStat(
+            name=lookup.entry.vfs_name or lookup.entry.name,
+            type=FileType.DIRECTORY,
+            extra={"guild_id": lookup.entry.id},
+        )
+
+    if len(parts) == 2 and parts[1] in VIRTUAL_DIRS:
+        return FileStat(name=parts[1], type=FileType.DIRECTORY)
+
+    if len(parts) == 3 and parts[1] == "channels":
+        if index is None:
+            raise enoent(virtual)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is None:
+            await _populate_via_parent(accessor, virtual_key, prefix, index)
+            lookup = await index.get(virtual_key)
+            if lookup.entry is None:
+                raise enoent(virtual)
+        return FileStat(
+            name=lookup.entry.vfs_name or lookup.entry.name,
+            type=FileType.DIRECTORY,
+            modified=snowflake_to_iso(lookup.entry.remote_time),
+            extra={"channel_id": lookup.entry.id},
+        )
+
+    if len(parts) == 3 and parts[1] == "members":
+        if index is None:
+            raise enoent(virtual)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is None:
+            await _populate_via_parent(accessor, virtual_key, prefix, index)
+            lookup = await index.get(virtual_key)
+            if lookup.entry is None:
+                raise enoent(virtual)
+        return FileStat(
+            name=lookup.entry.vfs_name or lookup.entry.name,
+            type=FileType.JSON,
+            extra={"user_id": lookup.entry.id},
+        )
+
+    # <guild>/channels/<ch>/<date>
+    if (len(parts) == 4 and parts[1] == "channels"
+            and _DATE_RE.match(parts[3])):
+        return FileStat(name=parts[3], type=FileType.DIRECTORY)
+
+    # <guild>/channels/<ch>/<date>/chat.jsonl
+    if (len(parts) == 5 and parts[1] == "channels" and _DATE_RE.match(parts[3])
+            and parts[4] == "chat.jsonl"):
+        return FileStat(name="chat.jsonl", type=FileType.TEXT)
+
+    # <guild>/channels/<ch>/<date>/files
+    if (len(parts) == 5 and parts[1] == "channels" and _DATE_RE.match(parts[3])
+            and parts[4] == "files"):
+        return FileStat(name="files", type=FileType.DIRECTORY)
+
+    # <guild>/channels/<ch>/<date>/files/<blob>
+    if (len(parts) == 6 and parts[1] == "channels" and _DATE_RE.match(parts[3])
+            and parts[4] == "files"):
+        if index is None:
+            raise enoent(virtual)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is None:
+            await _populate_via_parent(accessor, virtual_key, prefix, index)
+            lookup = await index.get(virtual_key)
+            if lookup.entry is None:
+                raise enoent(virtual)
+        mimetype = (lookup.entry.extra or {}).get("content_type", "")
+        return FileStat(
+            name=lookup.entry.vfs_name or lookup.entry.name,
+            size=lookup.entry.size,
+            type=filetype_from_mimetype(mimetype),
+            extra={
+                "content_type": mimetype,
+                "attachment_id": lookup.entry.id,
+            },
+        )
+
+    raise enoent(virtual)

python/mirage/core/disk/__init__.py

@@ -0,0 +1,50 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.core.disk.copy import copy
+from mirage.core.disk.create import create
+from mirage.core.disk.du import du, du_all
+from mirage.core.disk.exists import exists
+from mirage.core.disk.find import find
+from mirage.core.disk.mkdir import mkdir
+from mirage.core.disk.read import read_bytes
+from mirage.core.disk.readdir import readdir
+from mirage.core.disk.rename import rename
+from mirage.core.disk.rm import rm_r
+from mirage.core.disk.rmdir import rmdir
+from mirage.core.disk.stat import stat
+from mirage.core.disk.stream import read_stream
+from mirage.core.disk.truncate import truncate
+from mirage.core.disk.unlink import unlink
+from mirage.core.disk.write import write_bytes
+
+__all__ = [
+    "copy",
+    "create",
+    "du",
+    "du_all",
+    "exists",
+    "find",
+    "mkdir",
+    "read_bytes",
+    "read_stream",
+    "readdir",
+    "rename",
+    "rm_r",
+    "rmdir",
+    "stat",
+    "truncate",
+    "unlink",
+    "write_bytes",
+]

python/mirage/core/disk/append.py

@@ -0,0 +1,48 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import time
+from pathlib import Path
+
+import aiofiles
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.observe.context import record
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def append_bytes(accessor: DiskAccessor, path: PathSpec,
+                       data: bytes) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    root = accessor.root
+    start_ms = int(time.monotonic() * 1000)
+    p = _resolve(root, path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    async with aiofiles.open(p, "ab") as f:
+        await f.write(data)
+    record("append", path, "disk", len(data), start_ms)
+    await invalidate_after_write(path)

python/mirage/core/disk/constants.py

@@ -0,0 +1,16 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+SCOPE_WARN = 5000
+SCOPE_ERROR = 50000

python/mirage/core/disk/copy.py

@@ -0,0 +1,51 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import shutil
+from pathlib import Path
+
+import aiofiles.os
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def copy(accessor: DiskAccessor, src: PathSpec, dst: PathSpec) -> None:
+    if isinstance(src, str):
+        src = PathSpec(virtual=src,
+                       directory=src,
+                       resource_path=src.strip("/"))
+    if isinstance(src, PathSpec):
+        src = src.mount_path
+    if isinstance(dst, str):
+        dst = PathSpec(virtual=dst,
+                       directory=dst,
+                       resource_path=dst.strip("/"))
+    if isinstance(dst, PathSpec):
+        dst = dst.mount_path
+    root = accessor.root
+    s = _resolve(root, src)
+    d = _resolve(root, dst)
+    await aiofiles.os.makedirs(d.parent, exist_ok=True)
+    await asyncio.to_thread(shutil.copy2, s, d)
+    await invalidate_after_write(dst)

python/mirage/core/disk/create.py

@@ -0,0 +1,43 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles
+import aiofiles.os
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def create(accessor: DiskAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    await aiofiles.os.makedirs(p.parent, exist_ok=True)
+    async with aiofiles.open(p, "wb") as f:
+        await f.write(b"")
+    await invalidate_after_write(path)

python/mirage/core/disk/du.py

@@ -0,0 +1,86 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import os
+from pathlib import Path
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+def _du_sync(root: Path, path: str) -> int:
+    p = _resolve(root, path)
+    if p.is_file():
+        return p.stat().st_size
+    total = 0
+    for dirpath, _dirnames, filenames in os.walk(p):
+        for f in filenames:
+            try:
+                total += os.path.getsize(os.path.join(dirpath, f))
+            except OSError:
+                pass
+    return total
+
+
+def _du_all_sync(root: Path, path: str) -> tuple[list[tuple[str, int]], int]:
+    p = _resolve(root, path)
+    if p.is_file():
+        size = p.stat().st_size
+        return [(("/" + path.strip("/")), size)], size
+    entries: list[tuple[str, int]] = []
+    total = 0
+    for dirpath, _dirnames, filenames in os.walk(p):
+        for f in filenames:
+            full = os.path.join(dirpath, f)
+            try:
+                size = os.path.getsize(full)
+            except OSError:
+                continue
+            rel = os.path.relpath(full, root)
+            entry_path = "/" + rel
+            entries.append((entry_path, size))
+            total += size
+    entries.sort()
+    return entries, total
+
+
+async def du(accessor: DiskAccessor, path: PathSpec) -> int:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    return await asyncio.to_thread(_du_sync, accessor.root, path)
+
+
+async def du_all(
+    accessor: DiskAccessor,
+    path: PathSpec,
+) -> tuple[list[tuple[str, int]], int]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    return await asyncio.to_thread(_du_all_sync, accessor.root, path)

python/mirage/core/disk/exists.py

@@ -0,0 +1,38 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+from aiofiles.os import path as aio_path
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def exists(accessor: DiskAccessor, path: PathSpec) -> bool:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    return await aio_path.exists(p)

python/mirage/core/disk/find.py

@@ -0,0 +1,197 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
+                                               emit_start_path, keep,
+                                               start_basename)
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+def _find_sync(
+    root: Path,
+    path: str,
+    name: str | None = None,
+    type: str | None = None,
+    min_size: int | None = None,
+    max_size: int | None = None,
+    maxdepth: int | None = None,
+    name_exclude: str | None = None,
+    or_names: list[str] | None = None,
+    mtime_min: float | None = None,
+    mtime_max: float | None = None,
+    iname: str | None = None,
+    path_pattern: str | None = None,
+    mindepth: int | None = None,
+    empty: bool = False,
+    tree: PredNode | None = None,
+    start_name: str = "",
+) -> list[str]:
+    p = _resolve(root, path)
+    base = "/" + path.strip("/")
+    base_depth = 0 if base == "/" else base.count("/")
+    results: list[str] = []
+    tree = tree if tree is not None else build_tree(name=name,
+                                                    iname=iname,
+                                                    path_pattern=path_pattern,
+                                                    type=type,
+                                                    name_exclude=name_exclude,
+                                                    or_names=or_names,
+                                                    empty=empty)
+
+    if p.is_dir():
+        root_empty = (not any(p.iterdir())) if empty else None
+        emit_start_path(results,
+                        base,
+                        start_name,
+                        kind="d",
+                        is_empty=root_empty,
+                        exists=True,
+                        tree=tree,
+                        maxdepth=maxdepth,
+                        mindepth=mindepth,
+                        min_size=min_size,
+                        max_size=max_size)
+
+    for dirpath, dirnames, filenames in os.walk(p):
+        dp = Path(dirpath)
+        rel = dp.relative_to(root)
+        current = "/" + str(rel) if str(rel) != "." else "/"
+
+        current_depth = current.count("/") - base_depth
+
+        if maxdepth is not None and current_depth > maxdepth:
+            dirnames.clear()
+            continue
+
+        entries: list[tuple[str, str]] = []
+        if type != "f" and type != "file":
+            for d in dirnames:
+                entry_path = current.rstrip("/") + "/" + d
+                entries.append((entry_path, "d"))
+        if type != "d" and type != "directory":
+            for f in filenames:
+                entry_path = current.rstrip("/") + "/" + f
+                entries.append((entry_path, "f"))
+
+        for entry_path, kind in entries:
+            entry_name = entry_path.rsplit("/", 1)[-1]
+            depth = entry_path.count("/") - base_depth
+            if maxdepth is not None and depth > maxdepth:
+                continue
+
+            full = root / entry_path.lstrip("/")
+            is_empty: bool | None = None
+            if empty:
+                try:
+                    is_empty = (full.stat().st_size == 0) if kind == "f" else (
+                        not any(full.iterdir()))
+                except OSError:
+                    is_empty = None
+            entry = FindEntry(key=entry_path,
+                              name=entry_name,
+                              kind=kind,
+                              depth=depth,
+                              is_empty=is_empty)
+            if not keep(entry, tree, mindepth):
+                continue
+
+            # Directories count as size 0 for -size (deliberate GNU
+            # divergence).
+            if min_size is not None or max_size is not None:
+                if kind == "f":
+                    try:
+                        size = full.stat().st_size
+                    except OSError:
+                        continue
+                else:
+                    size = 0
+                if min_size is not None and size < min_size:
+                    continue
+                if max_size is not None and size > max_size:
+                    continue
+
+            if mtime_min is not None or mtime_max is not None:
+                try:
+                    st = full.stat()
+                    mtime = datetime.fromtimestamp(
+                        st.st_mtime, tz=timezone.utc).timestamp()
+                except OSError:
+                    continue
+                if mtime_min is not None and mtime < mtime_min:
+                    continue
+                if mtime_max is not None and mtime > mtime_max:
+                    continue
+
+            results.append(entry_path)
+
+    return sorted(results)
+
+
+async def find(
+    accessor: DiskAccessor,
+    path: PathSpec,
+    name: str | None = None,
+    type: str | None = None,
+    min_size: int | None = None,
+    max_size: int | None = None,
+    maxdepth: int | None = None,
+    name_exclude: str | None = None,
+    or_names: list[str] | None = None,
+    mtime_min: float | None = None,
+    mtime_max: float | None = None,
+    iname: str | None = None,
+    path_pattern: str | None = None,
+    mindepth: int | None = None,
+    empty: bool = False,
+    tree: PredNode | None = None,
+) -> list[str]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    start_name = start_basename(path)
+    path = path.mount_path
+    return await asyncio.to_thread(
+        _find_sync,
+        accessor.root,
+        path,
+        name,
+        type,
+        min_size,
+        max_size,
+        maxdepth,
+        name_exclude,
+        or_names,
+        mtime_min,
+        mtime_max,
+        iname,
+        path_pattern,
+        mindepth,
+        empty,
+        tree,
+        start_name,
+    )

python/mirage/core/disk/glob.py

@@ -0,0 +1,29 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.disk.constants import SCOPE_ERROR
+from mirage.core.disk.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: DiskAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/disk/mkdir.py

@@ -0,0 +1,48 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles.os
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def mkdir(accessor: DiskAccessor,
+                path: PathSpec,
+                parents: bool = False) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    if parents:
+        await aiofiles.os.makedirs(p, exist_ok=True)
+    else:
+        try:
+            await aiofiles.os.mkdir(p)
+        except FileExistsError:
+            pass
+    await invalidate_after_write(path)

python/mirage/core/disk/read.py

@@ -0,0 +1,52 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import time
+from pathlib import Path
+
+import aiofiles
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.observe.context import record
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def read_bytes(accessor: DiskAccessor,
+                     path: PathSpec,
+                     index: IndexCacheStore = None) -> bytes:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    root = accessor.root
+    start_ms = int(time.monotonic() * 1000)
+    p = _resolve(root, path)
+    try:
+        async with aiofiles.open(p, "rb") as f:
+            data = await f.read()
+    except FileNotFoundError as exc:
+        raise FileNotFoundError(virtual) from exc
+    record("read", path, "disk", len(data), start_ms)
+    return data

python/mirage/core/disk/readdir.py

@@ -0,0 +1,66 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles.os
+from aiofiles.os import path as aio_path
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.types import PathSpec
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def readdir(accessor: DiskAccessor, path: PathSpec,
+                  index: IndexCacheStore) -> list[str]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        prefix = mount_prefix_of(path.virtual, path.resource_path)
+        path = path.directory if path.pattern else path.virtual
+    if prefix and path.startswith(prefix):
+        rest = path[len(prefix):]
+        if prefix.endswith("/") or rest == "" or rest.startswith("/"):
+            path = rest or "/"
+    root = accessor.root
+    # Canonical key: no trailing slash (except root), or the same dir
+    # indexes under two keys and cache hits return doubled-slash entries.
+    virtual_key = prefix + path if prefix else path
+    virtual_key = virtual_key.rstrip("/") or "/"
+    listing = await index.list_dir(virtual_key)
+    if listing.entries is not None:
+        return listing.entries
+    p = _resolve(root, path)
+    if not await aio_path.isdir(p):
+        raise NotADirectoryError(str(p))
+    base = "/" + path.strip("/")
+    raw = await aiofiles.os.listdir(p)
+    entries = sorted(base.rstrip("/") + "/" + name for name in raw)
+    virtual_entries = sorted((prefix + e if prefix else e) for e in entries)
+    index_entries = [(e.rsplit("/", 1)[-1],
+                      IndexEntry(id=e,
+                                 name=e.rsplit("/", 1)[-1],
+                                 resource_type="file")) for e in entries]
+    await index.set_dir(virtual_key, index_entries)
+    return virtual_entries

python/mirage/core/disk/rename.py

@@ -0,0 +1,48 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles.os
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import (invalidate_after_unlink,
+                                  invalidate_after_write)
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def rename(accessor: DiskAccessor, src: PathSpec, dst: PathSpec) -> None:
+    if isinstance(src, str):
+        src = PathSpec(virtual=src,
+                       directory=src,
+                       resource_path=src.strip("/"))
+    if isinstance(src, PathSpec):
+        src = src.mount_path
+    if isinstance(dst, str):
+        dst = PathSpec(virtual=dst,
+                       directory=dst,
+                       resource_path=dst.strip("/"))
+    if isinstance(dst, PathSpec):
+        dst = dst.mount_path
+    root = accessor.root
+    await invalidate_after_unlink(src)
+    await invalidate_after_write(dst)
+    await aiofiles.os.rename(_resolve(root, src), _resolve(root, dst))

python/mirage/core/disk/rm.py

@@ -0,0 +1,46 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import shutil
+from pathlib import Path
+
+import aiofiles.os
+from aiofiles.os import path as aio_path
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def rm_r(accessor: DiskAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    if await aio_path.isdir(p):
+        await asyncio.to_thread(shutil.rmtree, p)
+    elif await aio_path.exists(p):
+        await aiofiles.os.remove(p)
+    await invalidate_after_unlink(path)

python/mirage/core/disk/rmdir.py

@@ -0,0 +1,40 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles.os
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def rmdir(accessor: DiskAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    await aiofiles.os.rmdir(p)
+    await invalidate_after_unlink(path)

python/mirage/core/disk/stat.py

@@ -0,0 +1,59 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles.os
+from aiofiles.os import path as aio_path
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.timeutil import epoch_to_iso
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.filetype import guess_type
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def stat(accessor: DiskAccessor,
+               path: PathSpec,
+               index: IndexCacheStore = None) -> FileStat:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    root = accessor.root
+    p = _resolve(root, path)
+    if not await aio_path.exists(p):
+        raise FileNotFoundError(virtual)
+    st = await aiofiles.os.stat(p)
+    modified = epoch_to_iso(st.st_mtime)
+    if await aio_path.isdir(p):
+        return FileStat(name=p.name,
+                        size=None,
+                        modified=modified,
+                        type=FileType.DIRECTORY)
+    return FileStat(name=p.name,
+                    size=st.st_size,
+                    modified=modified,
+                    fingerprint=modified,
+                    type=guess_type(p.name))

python/mirage/core/disk/stream.py

@@ -0,0 +1,57 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import AsyncIterator
+from pathlib import Path
+
+import aiofiles
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.observe.context import record_stream
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def read_stream(accessor: DiskAccessor,
+                      path: PathSpec,
+                      index: IndexCacheStore = None,
+                      chunk_size: int = 8192) -> AsyncIterator[bytes]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    root = accessor.root
+    rec = record_stream("read", path, "disk")
+    p = _resolve(root, path)
+    try:
+        async with aiofiles.open(p, "rb") as f:
+            while True:
+                chunk = await f.read(chunk_size)
+                if not chunk:
+                    break
+                if rec is not None:
+                    rec.bytes += len(chunk)
+                yield chunk
+    except FileNotFoundError as exc:
+        raise FileNotFoundError(virtual) from exc

python/mirage/core/disk/truncate.py

@@ -0,0 +1,48 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def truncate(accessor: DiskAccessor, path: PathSpec,
+                   length: int) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    try:
+        async with aiofiles.open(p, "rb") as f:
+            data = await f.read()
+    except FileNotFoundError:
+        data = b""
+    result = data[:length].ljust(length, b"\0")
+    async with aiofiles.open(p, "wb") as f:
+        await f.write(result)
+    await invalidate_after_write(path)

python/mirage/core/disk/unlink.py

@@ -0,0 +1,40 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pathlib import Path
+
+import aiofiles.os
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def unlink(accessor: DiskAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    p = _resolve(accessor.root, path)
+    await aiofiles.os.remove(p)
+    await invalidate_after_unlink(path)

python/mirage/core/disk/write.py

@@ -0,0 +1,48 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import time
+from pathlib import Path
+
+import aiofiles
+
+from mirage.accessor.disk import DiskAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.observe.context import record
+from mirage.types import PathSpec
+
+
+def _resolve(root: Path, path: str) -> Path:
+    relative = path.lstrip("/")
+    resolved = (root / relative).resolve()
+    resolved.relative_to(root)
+    return resolved
+
+
+async def write_bytes(accessor: DiskAccessor, path: PathSpec,
+                      data: bytes) -> None:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    if isinstance(path, PathSpec):
+        path = path.mount_path
+    root = accessor.root
+    start_ms = int(time.monotonic() * 1000)
+    p = _resolve(root, path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    async with aiofiles.open(p, "wb") as f:
+        await f.write(data)
+    record("write", path, "disk", len(data), start_ms)
+    await invalidate_after_write(path)

python/mirage/core/email/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/mirage/core/email/_client.py

@@ -0,0 +1,211 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from email import policy
+from email.parser import BytesParser
+
+from mirage.accessor.email import EmailAccessor
+from mirage.core.email._parse import parse_rfc822
+
+
+async def list_folders(accessor: EmailAccessor) -> list[str]:
+    imap = await accessor.get_imap()
+    response = await imap.list('""', "*")
+    folders: list[str] = []
+    for line in response.lines:
+        if isinstance(line, (bytes, bytearray)):
+            line = bytes(line).decode(errors="replace")
+        if '"' in line:
+            parts = line.rsplit('"', 2)
+            if len(parts) >= 2:
+                folders.append(parts[-2])
+    return folders
+
+
+async def list_message_uids(
+    accessor: EmailAccessor,
+    folder: str,
+    search_criteria: str = "ALL",
+    max_results: int | None = None,
+) -> list[str]:
+    imap = await accessor.get_imap()
+    await imap.select(folder)
+    response = await imap.search(search_criteria, charset=None)
+    if response.result != "OK" or not response.lines:
+        return []
+    raw = response.lines[0]
+    if isinstance(raw, (bytes, bytearray)):
+        raw = bytes(raw).decode()
+    seq_nums = raw.split() if raw.strip() else []
+    if not seq_nums:
+        return []
+    if max_results is not None:
+        seq_nums = seq_nums[-max_results:]
+    uids: list[str] = []
+    batch_size = 50
+    for i in range(0, len(seq_nums), batch_size):
+        batch = seq_nums[i:i + batch_size]
+        seq_set = ",".join(batch)
+        uid_response = await imap.fetch(seq_set, "(UID)")
+        for item in uid_response.lines:
+            if isinstance(item, (bytes, bytearray)):
+                line = bytes(item).decode(errors="replace")
+            else:
+                line = str(item)
+            if "UID" in line:
+                try:
+                    uid_idx = line.index("UID") + 4
+                    rest = line[uid_idx:].strip()
+                    uid_val = rest.split(")")[0].split()[0]
+                    uids.append(uid_val)
+                except (ValueError, IndexError):
+                    pass
+    return uids
+
+
+async def fetch_message(
+    accessor: EmailAccessor,
+    folder: str,
+    uid: str,
+) -> dict:
+    imap = await accessor.get_imap()
+    await imap.select(folder)
+    response = await imap.uid("fetch", uid, "(RFC822 FLAGS)")
+    raw_bytes = _extract_body(response)
+    flags = _extract_flags(response)
+    msg_dict = parse_rfc822(raw_bytes)
+    msg_dict["uid"] = uid
+    msg_dict["flags"] = flags
+    return msg_dict
+
+
+async def fetch_headers(
+    accessor: EmailAccessor,
+    folder: str,
+    uids: list[str],
+) -> list[dict]:
+    if not uids:
+        return []
+    imap = await accessor.get_imap()
+    await imap.select(folder)
+    results: list[dict] = []
+    batch_size = 25
+    for i in range(0, len(uids), batch_size):
+        batch = uids[i:i + batch_size]
+        uid_set = ",".join(batch)
+        response = await imap.uid("fetch", uid_set, "(BODY[HEADER] FLAGS UID)")
+        results.extend(_parse_multi_fetch(response, batch))
+    return results
+
+
+async def fetch_attachment(
+    accessor: EmailAccessor,
+    folder: str,
+    uid: str,
+    filename: str,
+) -> bytes | None:
+    imap = await accessor.get_imap()
+    await imap.select(folder)
+    response = await imap.uid("fetch", uid, "(RFC822)")
+    raw_bytes = _extract_body(response)
+    attachments = _parse_with_payloads(raw_bytes)
+    for att in attachments:
+        if att["filename"] == filename:
+            return att["payload"]
+    return None
+
+
+def _parse_with_payloads(raw: bytes) -> list[dict]:
+    msg = BytesParser(policy=policy.default).parsebytes(raw)
+    attachments: list[dict] = []
+    if msg.is_multipart():
+        for part in msg.walk():
+            disposition = str(part.get("Content-Disposition", ""))
+            if "attachment" in disposition:
+                payload = part.get_payload(decode=True) or b""
+                attachments.append({
+                    "filename": part.get_filename() or "unnamed",
+                    "payload": payload,
+                })
+    return attachments
+
+
+def _extract_body(response) -> bytes:
+    for item in response.lines:
+        if isinstance(item, (bytearray, )) and len(item) > 20:
+            return bytes(item)
+        if isinstance(item, bytes) and len(item) > 100:
+            return item
+    return b""
+
+
+def _extract_flags(response) -> list[str]:
+    for item in response.lines:
+        if isinstance(item, (bytes, bytearray)):
+            line = bytes(item).decode(errors="replace")
+        else:
+            line = str(item)
+        if "FLAGS" in line:
+            try:
+                start = line.index("(", line.index("FLAGS")) + 1
+                end = line.index(")", start)
+                return line[start:end].split()
+            except ValueError:
+                pass
+    return []
+
+
+def _parse_multi_fetch(response, uids: list[str]) -> list[dict]:
+    results: list[dict] = []
+    current_uid = None
+    current_flags: list[str] = []
+
+    for item in response.lines:
+        if isinstance(item, (bytes, bytearray)):
+            line = bytes(item).decode(errors="replace")
+        else:
+            line = str(item)
+
+        if "FETCH" in line and "UID" in line:
+            try:
+                uid_idx = line.index("UID") + 4
+                uid_end = line.index(
+                    " ", uid_idx) if " " in line[uid_idx:] else len(line)
+                current_uid = line[uid_idx:uid_end].strip(")")
+            except (ValueError, IndexError):
+                pass
+            if "FLAGS" in line:
+                current_flags = _extract_flags_from_line(line)
+            continue
+
+        if isinstance(item, (bytearray, )) and len(item) > 20:
+            raw = bytes(item)
+            msg_dict = parse_rfc822(raw, headers_only=True)
+            msg_dict["uid"] = current_uid or (uids[len(results)] if
+                                              len(results) < len(uids) else "")
+            msg_dict["flags"] = current_flags
+            results.append(msg_dict)
+            current_uid = None
+            current_flags = []
+
+    return results
+
+
+def _extract_flags_from_line(line: str) -> list[str]:
+    try:
+        start = line.index("(", line.index("FLAGS")) + 1
+        end = line.index(")", start)
+        return line[start:end].split()
+    except ValueError:
+        return []

python/mirage/core/email/_parse.py

@@ -0,0 +1,99 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from email import policy
+from email.parser import BytesParser
+
+
+def parse_rfc822(raw: bytes, headers_only: bool = False) -> dict:
+    parser = BytesParser(policy=policy.default)
+    if headers_only:
+        msg = parser.parsebytes(raw, headersonly=True)
+    else:
+        msg = parser.parsebytes(raw)
+
+    body_text, body_html = "", ""
+    attachments: list[dict] = []
+
+    if not headers_only:
+        body_text, body_html, attachments = _extract_parts(msg)
+
+    return {
+        "from": _parse_address(msg.get("From", "")),
+        "to": _parse_address_list(msg.get("To", "")),
+        "cc": _parse_address_list(msg.get("Cc", "")),
+        "subject": msg.get("Subject", ""),
+        "date": msg.get("Date", ""),
+        "body_text": body_text,
+        "body_html": body_html,
+        "snippet": body_text[:100],
+        "message_id": msg.get("Message-ID", ""),
+        "in_reply_to": msg.get("In-Reply-To") or None,
+        "references": (msg.get("References") or "").split(),
+        "has_attachments": bool(attachments),
+        "attachments": attachments,
+    }
+
+
+def _extract_parts(msg) -> tuple[str, str, list[dict]]:
+    text, html = "", ""
+    attachments: list[dict] = []
+
+    if msg.is_multipart():
+        for part in msg.walk():
+            content_type = part.get_content_type()
+            disposition = str(part.get("Content-Disposition", ""))
+
+            if "attachment" in disposition:
+                payload = part.get_payload(decode=True) or b""
+                attachments.append({
+                    "filename": part.get_filename() or "unnamed",
+                    "content_type": content_type,
+                    "size": len(payload),
+                })
+            elif content_type == "text/plain" and not text:
+                text = (part.get_payload(decode=True)
+                        or b"").decode(part.get_content_charset() or "utf-8",
+                                       errors="replace")
+            elif content_type == "text/html" and not html:
+                html = (part.get_payload(decode=True)
+                        or b"").decode(part.get_content_charset() or "utf-8",
+                                       errors="replace")
+    else:
+        content_type = msg.get_content_type()
+        payload = msg.get_payload(decode=True) or b""
+        decoded = payload.decode(msg.get_content_charset() or "utf-8",
+                                 errors="replace")
+        if content_type == "text/html":
+            html = decoded
+        else:
+            text = decoded
+
+    return text, html, attachments
+
+
+def _parse_address(raw: str) -> dict[str, str]:
+    if not raw:
+        return {"name": "", "email": ""}
+    if "<" in raw and ">" in raw:
+        name = raw[:raw.index("<")].strip().strip('"')
+        email = raw[raw.index("<") + 1:raw.index(">")]
+        return {"name": name, "email": email}
+    return {"name": "", "email": raw.strip()}
+
+
+def _parse_address_list(raw: str) -> list[dict[str, str]]:
+    if not raw:
+        return []
+    return [_parse_address(a.strip()) for a in raw.split(",")]

python/mirage/core/email/folders.py

@@ -0,0 +1,20 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.email import EmailAccessor
+from mirage.core.email._client import list_folders as _list_folders
+
+
+async def list_folders(accessor: EmailAccessor) -> list[str]:
+    return await _list_folders(accessor)

python/mirage/core/email/glob.py

@@ -0,0 +1,29 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.accessor.email import EmailAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.commands.builtin.constants import SCOPE_ERROR
+from mirage.core.email.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: EmailAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/email/read.py

@@ -0,0 +1,64 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import json
+import posixpath
+
+from mirage.accessor.email import EmailAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.email._client import fetch_attachment, fetch_message
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+async def read(
+    accessor: EmailAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> bytes:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    key = path.resource_path
+    if index is None:
+        raise enoent(virtual)
+    virtual_key = prefix + "/" + key if prefix else "/" + key
+    result = await index.get(virtual_key)
+    if result.entry is None:
+        raise enoent(virtual)
+    if result.entry.resource_type in ("email/folder", "email/date",
+                                      "email/attachment_dir"):
+        raise IsADirectoryError(virtual)
+    if result.entry.resource_type == "email/attachment":
+        parent_key = posixpath.dirname(virtual_key)
+        parent_result = await index.get(parent_key)
+        if parent_result.entry is None:
+            raise enoent(virtual)
+        uid = parent_result.entry.id
+        parts = virtual_key.strip("/").split("/")
+        folder = parts[1] if prefix else parts[0]
+        filename = result.entry.vfs_name
+        data = await fetch_attachment(accessor, folder, uid, filename)
+        if data is None:
+            raise enoent(virtual)
+        return data
+    parts = virtual_key.strip("/").split("/")
+    folder = parts[1] if prefix else parts[0]
+    uid = result.entry.id
+    msg = await fetch_message(accessor, folder, uid)
+    return json.dumps(msg, ensure_ascii=False, separators=(",", ":")).encode()

python/mirage/core/email/readdir.py

@@ -0,0 +1,195 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import re
+from email.utils import parsedate_to_datetime
+
+from mirage.accessor.email import EmailAccessor
+from mirage.cache.index import IndexCacheStore, IndexEntry
+from mirage.core.email._client import fetch_headers, list_message_uids
+from mirage.core.email.folders import list_folders
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+TITLE_MAX = 80
+UNSAFE = re.compile(r"[^\w\s\-.]")
+MULTI_UNDERSCORE = re.compile(r"_+")
+
+
+def _sanitize(text: str) -> str:
+    if not text.strip():
+        return "No_Subject"
+    cleaned = UNSAFE.sub("_", text).replace(" ", "_")
+    cleaned = MULTI_UNDERSCORE.sub("_", cleaned).strip("_")
+    if len(cleaned) > TITLE_MAX:
+        cleaned = cleaned[:TITLE_MAX - 3] + "..."
+    return cleaned
+
+
+def _msg_filename(subject: str, uid: str) -> str:
+    return f"{_sanitize(subject)}__{uid}.email.json"
+
+
+def _date_from_header(date_str: str) -> str:
+    try:
+        dt = parsedate_to_datetime(date_str)
+        return dt.strftime("%Y-%m-%d")
+    except Exception:
+        return "1970-01-01"
+
+
+async def readdir(
+    accessor: EmailAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> list[str]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    path = (path.dir if path.pattern else path).mount_path
+    key = path.strip("/")
+    virtual_key = prefix + "/" + key if key else prefix or "/"
+    parts = key.split("/") if key else []
+    depth = len(parts)
+
+    if depth == 0:
+        if index is not None:
+            cached = await index.list_dir(virtual_key)
+            if cached.entries is not None:
+                return cached.entries
+        folders = await list_folders(accessor)
+        entries = []
+        for folder_name in folders:
+            entry = IndexEntry(
+                id=folder_name,
+                name=folder_name,
+                resource_type="email/folder",
+                vfs_name=folder_name,
+            )
+            entries.append((folder_name, entry))
+        if index is not None:
+            await index.set_dir(virtual_key, entries)
+        return [f"{prefix}/{name}" for name, _ in entries]
+
+    if depth == 1:
+        folder_name = parts[0]
+        if index is not None:
+            cached = await index.list_dir(virtual_key)
+            if cached.entries is not None:
+                return cached.entries
+        if index is None:
+            raise enoent(virtual)
+        max_msgs = accessor.config.max_messages
+        uids = await list_message_uids(accessor,
+                                       folder_name,
+                                       max_results=max_msgs)
+        headers_list = await fetch_headers(accessor, folder_name, uids)
+        date_groups: dict[str, list[dict]] = {}
+        for hdr in headers_list:
+            date_str = _date_from_header(hdr.get("date", ""))
+            date_groups.setdefault(date_str, []).append(hdr)
+        date_entries: list[tuple[str, IndexEntry]] = []
+        for date_str in sorted(date_groups.keys(), reverse=True):
+            date_entry = IndexEntry(
+                id=date_str,
+                name=date_str,
+                resource_type="email/date",
+                vfs_name=date_str,
+            )
+            date_entries.append((date_str, date_entry))
+            msg_entries: list[tuple[str, IndexEntry]] = []
+            for hdr in date_groups[date_str]:
+                uid = hdr["uid"]
+                subject = hdr.get("subject", "") or "No Subject"
+                filename = _msg_filename(subject, uid)
+                msg_entry = IndexEntry(
+                    id=uid,
+                    name=subject,
+                    resource_type="email/message",
+                    vfs_name=filename,
+                )
+                msg_entries.append((filename, msg_entry))
+                attachments = hdr.get("attachments", [])
+                if attachments:
+                    att_dir_name = filename.replace(".email.json", "")
+                    att_dir_entry = IndexEntry(
+                        id=uid,
+                        name=att_dir_name,
+                        resource_type="email/attachment_dir",
+                        vfs_name=att_dir_name,
+                    )
+                    msg_entries.append((att_dir_name, att_dir_entry))
+                    att_entries: list[tuple[str, IndexEntry]] = []
+                    for att in attachments:
+                        att_entry = IndexEntry(
+                            id=att["filename"],
+                            name=att["filename"],
+                            resource_type="email/attachment",
+                            vfs_name=att["filename"],
+                            size=att.get("size"),
+                        )
+                        att_entries.append((att["filename"], att_entry))
+                    att_dir_vkey = (virtual_key + "/" + date_str + "/" +
+                                    att_dir_name)
+                    await index.set_dir(att_dir_vkey, att_entries)
+            date_vkey = virtual_key + "/" + date_str
+            await index.set_dir(date_vkey, msg_entries)
+        await index.set_dir(virtual_key, date_entries)
+        return [f"{prefix}/{key}/{name}" for name, _ in date_entries]
+
+    if depth == 2:
+        if index is None:
+            raise enoent(virtual)
+        cached = await index.list_dir(virtual_key)
+        if cached.entries is not None:
+            return cached.entries
+        folder_vkey = PathSpec(
+            virtual=prefix + "/" + parts[0],
+            directory=prefix + "/" + parts[0],
+            resource_path=mount_key(prefix + "/" + parts[0], prefix),
+        )
+        await readdir(accessor, folder_vkey, index)
+        cached = await index.list_dir(virtual_key)
+        if cached.entries is not None:
+            return cached.entries
+        raise enoent(virtual)
+
+    if depth == 3:
+        if index is None:
+            raise enoent(virtual)
+        cached = await index.list_dir(virtual_key)
+        if cached.entries is not None:
+            return cached.entries
+        folder_vkey = PathSpec(
+            virtual=prefix + "/" + parts[0],
+            directory=prefix + "/" + parts[0],
+            resource_path=mount_key(prefix + "/" + parts[0], prefix),
+        )
+        await readdir(accessor, folder_vkey, index)
+        cached = await index.list_dir(virtual_key)
+        if cached.entries is not None:
+            return cached.entries
+        raise enoent(virtual)
+
+    raise enoent(virtual)
+
+
+def is_dir_name(child: str) -> bool:
+    # Entries are recognized by extension, so classification never needs
+    # the stat fallback.
+    return not child.rsplit("/", 1)[-1].endswith(".email.json")

python/mirage/core/email/scope.py

@@ -0,0 +1,63 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from dataclasses import dataclass
+
+from mirage.types import PathSpec
+
+
+@dataclass
+class EmailScope:
+    use_native: bool
+    folder: str | None = None
+    resource_path: str = "/"
+
+
+def detect_scope(path: PathSpec) -> EmailScope:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    key = path.mount_path.strip("/")
+    if not key:
+        return EmailScope(use_native=False, resource_path="/")
+    parts = [x for x in key.split("/") if x]
+    if not parts:
+        return EmailScope(use_native=False, resource_path="/")
+    if key.endswith(".email.json"):
+        return EmailScope(
+            use_native=False,
+            folder=parts[0],
+            resource_path=key,
+        )
+    if len(parts) <= 2:
+        return EmailScope(
+            use_native=True,
+            folder=parts[0],
+            resource_path=key,
+        )
+    return EmailScope(
+        use_native=False,
+        folder=parts[0],
+        resource_path=key,
+    )
+
+
+def extract_folder(paths: list[PathSpec]) -> str | None:
+    if not paths:
+        return None
+    p = paths[0]
+    key = p.mount_path.strip("/")
+    parts = [x for x in key.split("/") if x]
+    return parts[0] if parts else None

python/mirage/core/email/search.py

@@ -0,0 +1,107 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import json
+
+from mirage.accessor.email import EmailAccessor
+from mirage.core.email._client import fetch_message, list_message_uids
+from mirage.core.email.readdir import _date_from_header, _sanitize
+from mirage.core.email.scope import EmailScope
+
+
+def build_search_criteria(
+    text: str | None = None,
+    subject: str | None = None,
+    from_addr: str | None = None,
+    to_addr: str | None = None,
+    since: str | None = None,
+    before: str | None = None,
+    unseen: bool = False,
+) -> str:
+    parts: list[str] = []
+    if unseen:
+        parts.append("UNSEEN")
+    if text:
+        parts.append(f'TEXT "{text}"')
+    if subject:
+        parts.append(f'SUBJECT "{subject}"')
+    if from_addr:
+        parts.append(f'FROM "{from_addr}"')
+    if to_addr:
+        parts.append(f'TO "{to_addr}"')
+    if since:
+        parts.append(f"SINCE {since}")
+    if before:
+        parts.append(f"BEFORE {before}")
+    return " ".join(parts) if parts else "ALL"
+
+
+async def search_messages(
+    accessor: EmailAccessor,
+    folder: str,
+    text: str | None = None,
+    subject: str | None = None,
+    from_addr: str | None = None,
+    to_addr: str | None = None,
+    since: str | None = None,
+    before: str | None = None,
+    unseen: bool = False,
+    max_results: int | None = None,
+) -> list[str]:
+    criteria = build_search_criteria(
+        text=text,
+        subject=subject,
+        from_addr=from_addr,
+        to_addr=to_addr,
+        since=since,
+        before=before,
+        unseen=unseen,
+    )
+    return await list_message_uids(accessor,
+                                   folder,
+                                   criteria,
+                                   max_results=max_results)
+
+
+def _build_vfs_path(prefix: str, folder: str, msg: dict) -> str:
+    date_str = _date_from_header(msg.get("date", ""))
+    subject = _sanitize(msg.get("subject", "No Subject"))
+    uid = msg.get("uid", "")
+    filename = f"{subject}__{uid}.email.json"
+    parts = [prefix, folder, date_str, filename]
+    return "/".join(p for p in parts if p)
+
+
+async def search_and_format(
+    accessor: EmailAccessor,
+    scope: EmailScope,
+    pattern: str,
+    prefix: str,
+    max_results: int | None = None,
+) -> list[tuple[str, str]]:
+    """Run native search and return (vfs_path, message_json) pairs."""
+    folder = scope.folder or ""
+    if not folder:
+        return []
+    uids = await search_messages(accessor,
+                                 folder,
+                                 text=pattern,
+                                 max_results=max_results)
+    pairs: list[tuple[str, str]] = []
+    for uid in uids:
+        msg = await fetch_message(accessor, folder, uid)
+        msg_text = json.dumps(msg, ensure_ascii=False, separators=(",", ":"))
+        vfs_path = _build_vfs_path(prefix, folder, msg)
+        pairs.append((vfs_path, msg_text))
+    return pairs

python/mirage/core/email/send.py

@@ -0,0 +1,118 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from email.message import EmailMessage
+
+import aiosmtplib
+
+from mirage.resource.email.config import EmailConfig
+from mirage.resource.secrets import reveal_secret
+
+
+async def _smtp_send(config: EmailConfig, msg: EmailMessage) -> None:
+    await aiosmtplib.send(
+        msg,
+        hostname=config.smtp_host,
+        port=config.smtp_port,
+        username=config.username,
+        password=reveal_secret(config.password),
+        start_tls=True,
+    )
+
+
+async def send_message(
+    config: EmailConfig,
+    to: str,
+    subject: str,
+    body: str,
+) -> dict:
+    msg = EmailMessage()
+    msg["From"] = config.username
+    msg["To"] = to
+    msg["Subject"] = subject
+    msg.set_content(body)
+    await _smtp_send(config, msg)
+    return {"status": "sent", "to": to, "subject": subject}
+
+
+async def reply_message(
+    config: EmailConfig,
+    original: dict,
+    body: str,
+) -> dict:
+    msg = EmailMessage()
+    msg["From"] = config.username
+    msg["To"] = original["from"]["email"]
+    msg["Subject"] = f"Re: {original['subject']}"
+    msg["In-Reply-To"] = original["message_id"]
+    refs = list(original.get("references", []))
+    refs.append(original["message_id"])
+    msg["References"] = " ".join(refs)
+    msg.set_content(body)
+    await _smtp_send(config, msg)
+    return {
+        "status": "sent",
+        "to": original["from"]["email"],
+        "subject": msg["Subject"],
+    }
+
+
+async def reply_all_message(
+    config: EmailConfig,
+    original: dict,
+    body: str,
+) -> dict:
+    all_recipients: set[str] = set()
+    all_recipients.add(original["from"]["email"])
+    for r in original.get("to", []):
+        all_recipients.add(r["email"])
+    for r in original.get("cc", []):
+        all_recipients.add(r["email"])
+    all_recipients.discard(config.username)
+
+    msg = EmailMessage()
+    msg["From"] = config.username
+    msg["To"] = ", ".join(sorted(all_recipients))
+    msg["Subject"] = f"Re: {original['subject']}"
+    msg["In-Reply-To"] = original["message_id"]
+    refs = list(original.get("references", []))
+    refs.append(original["message_id"])
+    msg["References"] = " ".join(refs)
+    msg.set_content(body)
+    await _smtp_send(config, msg)
+    return {
+        "status": "sent",
+        "to": sorted(all_recipients),
+        "subject": msg["Subject"],
+    }
+
+
+async def forward_message(
+    config: EmailConfig,
+    original: dict,
+    to: str,
+) -> dict:
+    msg = EmailMessage()
+    msg["From"] = config.username
+    msg["To"] = to
+    msg["Subject"] = f"Fwd: {original['subject']}"
+    fwd_body = (
+        f"---------- Forwarded message ----------\n"
+        f"From: {original['from']['name']} <{original['from']['email']}>\n"
+        f"Date: {original['date']}\n"
+        f"Subject: {original['subject']}\n\n"
+        f"{original['body_text']}")
+    msg.set_content(fwd_body)
+    await _smtp_send(config, msg)
+    return {"status": "sent", "to": to, "subject": msg["Subject"]}

python/mirage/core/email/stat.py

@@ -0,0 +1,107 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mimetypes import guess_type as _guess_mime
+
+from mirage.accessor.email import EmailAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.email._client import list_folders
+from mirage.core.email.readdir import readdir as _readdir
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.filetype import filetype_from_mimetype
+from mirage.utils.key_prefix import mount_key, mount_prefix_of
+
+
+def _guess_filetype(filename: str) -> FileType:
+    mime, _ = _guess_mime(filename)
+    return filetype_from_mimetype(mime or "")
+
+
+async def stat(
+    accessor: EmailAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+) -> FileStat:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    virtual = path.virtual
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    key = path.resource_path
+    if not key:
+        return FileStat(name="/", type=FileType.DIRECTORY)
+    if index is None:
+        raise enoent(virtual)
+    virtual_key = prefix + "/" + key if prefix else "/" + key
+    result = await index.get(virtual_key)
+    if result.entry is None and "/" not in key:
+        folders = await list_folders(accessor)
+        if key in folders:
+            return FileStat(name=key, type=FileType.DIRECTORY)
+        raise enoent(virtual)
+    if result.entry is None:
+        parent_virtual = virtual_key.rsplit("/", 1)[0] or "/"
+        try:
+            await _readdir(
+                accessor,
+                PathSpec(virtual=parent_virtual,
+                         directory=parent_virtual,
+                         resource_path=mount_key(parent_virtual, prefix)),
+                index=index,
+            )
+        # best-effort cache populate; canonical ENOENT raised below
+        except Exception:
+            pass
+        result = await index.get(virtual_key)
+        if result.entry is None:
+            raise enoent(virtual)
+    rt = result.entry.resource_type
+    if rt == "email/folder":
+        return FileStat(
+            name=result.entry.vfs_name,
+            type=FileType.DIRECTORY,
+        )
+    if rt == "email/date":
+        return FileStat(
+            name=result.entry.vfs_name,
+            type=FileType.DIRECTORY,
+        )
+    if rt == "email/message":
+        return FileStat(
+            name=result.entry.vfs_name,
+            type=FileType.JSON,
+            size=result.entry.size,
+            extra={"uid": result.entry.id},
+        )
+    if rt == "email/attachment_dir":
+        return FileStat(
+            name=result.entry.vfs_name,
+            type=FileType.DIRECTORY,
+            extra={"uid": result.entry.id},
+        )
+    if rt == "email/attachment":
+        ft = _guess_filetype(result.entry.vfs_name)
+        return FileStat(
+            name=result.entry.vfs_name,
+            type=ft,
+            size=result.entry.size,
+            extra={"attachment_id": result.entry.id},
+        )
+    return FileStat(
+        name=result.entry.vfs_name,
+        type=FileType.JSON,
+        extra={"uid": result.entry.id},
+    )

python/mirage/core/filetype/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/mirage/core/filetype/feather.py

@@ -0,0 +1,138 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import io
+import re
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.feather as feather
+
+_MAX_PREVIEW_ROWS = 20
+
+
+def _read_table(raw: bytes) -> pa.Table:
+    return feather.read_table(io.BytesIO(raw))
+
+
+def _render_schema(schema: pa.Schema) -> list[str]:
+    lines = ["## Schema"]
+    for field in schema:
+        lines.append(f"  {field.name}: {field.type}")
+    return lines
+
+
+def _render_table(table: pa.Table, label: str, count: int) -> list[str]:
+    lines = [f"## {label} ({count} rows)", ""]
+    lines.append(table.to_pandas().to_string(index=False))
+    lines.append("")
+    return lines
+
+
+def cat(raw: bytes, max_rows: int = _MAX_PREVIEW_ROWS) -> bytes:
+    table = _read_table(raw)
+    schema = table.schema
+    num_rows = table.num_rows
+    preview = table.slice(0, max_rows)
+    preview_count = min(num_rows, max_rows)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(preview, "Preview", preview_count))
+    return "\n".join(lines).encode()
+
+
+def head(raw: bytes, n: int = 10) -> bytes:
+    table = _read_table(raw)
+    schema = table.schema
+    num_rows = table.num_rows
+    rows_needed = min(n, num_rows)
+    result = table.slice(0, rows_needed)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(result, f"First {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def tail(raw: bytes, n: int = 10) -> bytes:
+    table = _read_table(raw)
+    schema = table.schema
+    num_rows = table.num_rows
+    rows_needed = min(n, num_rows)
+    result = table.slice(max(0, num_rows - rows_needed), rows_needed)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(result, f"Last {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def wc(raw: bytes) -> int:
+    table = _read_table(raw)
+    return table.num_rows
+
+
+def stat(raw: bytes) -> bytes:
+    table = _read_table(raw)
+    schema = table.schema
+    num_rows = table.num_rows
+    lines = [
+        "# Feather file",
+        f"rows: {num_rows}",
+        f"columns: {len(schema)}",
+        "",
+    ]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    return "\n".join(lines).encode()
+
+
+def grep(raw: bytes, pattern: str, ignore_case: bool = False) -> bytes:
+    flags = re.IGNORECASE if ignore_case else 0
+    regex = re.compile(pattern, flags)
+    table = _read_table(raw)
+    df = table.to_pandas()
+    str_cols = df.select_dtypes(include=["object", "string"]).columns
+    if len(str_cols) == 0:
+        return df.head(0).to_csv(index=False).encode()
+    row_mask = pd.Series(False, index=df.index)
+    for col_name in str_cols:
+        row_mask = row_mask | df[col_name].astype(str).str.contains(regex,
+                                                                    na=False)
+    matched = df[row_mask]
+    return matched.to_csv(index=False).encode()
+
+
+def cut(raw: bytes, columns: list[str]) -> bytes:
+    table = _read_table(raw)
+    schema_names = [f.name for f in table.schema]
+    for col in columns:
+        if col not in schema_names:
+            raise ValueError(f"column not found: {col}")
+    result = table.select(columns)
+    return result.to_pandas().to_csv(index=False).encode()
+
+
+def file(raw: bytes) -> bytes:
+    table = _read_table(raw)
+    schema = table.schema
+    cols = ", ".join(f"{f.name}: {f.type}" for f in schema)
+    return (f"feather, {table.num_rows} rows, {len(schema)} columns"
+            f" ({cols})").encode()
+
+
+def ls(raw: bytes) -> tuple[int, int]:
+    table = _read_table(raw)
+    return table.num_rows, len(table.schema)

python/mirage/core/filetype/hdf5.py

@@ -0,0 +1,135 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import re
+import tempfile
+
+import h5py
+import pandas as pd
+
+_MAX_PREVIEW_ROWS = 20
+
+
+def _read_df(raw: bytes) -> pd.DataFrame:
+    with tempfile.NamedTemporaryFile(suffix=".h5", delete=False) as f:
+        f.write(raw)
+        tmp = f.name
+    try:
+        store = pd.HDFStore(tmp, mode="r")
+        try:
+            keys = store.keys()
+            if not keys:
+                raise ValueError("no datasets found in HDF5 file")
+            return store[keys[0]]
+        finally:
+            store.close()
+    except Exception:
+        with h5py.File(tmp, "r") as hf:
+            keys = list(hf.keys())
+            if not keys:
+                raise ValueError("no datasets found in HDF5 file")
+            dset = hf[keys[0]]
+            if hasattr(dset, "shape") and len(dset.shape) == 2:
+                return pd.DataFrame(dset[:])
+            if hasattr(dset, "dtype") and dset.dtype.names:
+                return pd.DataFrame(dset[:])
+            raise ValueError("unsupported HDF5 dataset structure")
+
+
+def _render_schema(df: pd.DataFrame) -> list[str]:
+    lines = ["## Schema"]
+    for col in df.columns:
+        lines.append(f"  {col}: {df[col].dtype}")
+    return lines
+
+
+def _render_df(df: pd.DataFrame, label: str, count: int) -> list[str]:
+    lines = [f"## {label} ({count} rows)", ""]
+    lines.append(df.to_string(index=False))
+    lines.append("")
+    return lines
+
+
+def cat(raw: bytes, max_rows: int = _MAX_PREVIEW_ROWS) -> bytes:
+    df = _read_df(raw)
+    num_rows = len(df)
+    preview_count = min(num_rows, max_rows)
+    lines = [f"# Rows: {num_rows}, Columns: {len(df.columns)}", ""]
+    lines.extend(_render_schema(df))
+    lines.append("")
+    lines.extend(_render_df(df.head(max_rows), "Preview", preview_count))
+    return "\n".join(lines).encode()
+
+
+def head(raw: bytes, n: int = 10) -> bytes:
+    df = _read_df(raw)
+    num_rows = len(df)
+    rows_needed = min(n, num_rows)
+    lines = [f"# Rows: {num_rows}, Columns: {len(df.columns)}", ""]
+    lines.extend(_render_schema(df))
+    lines.append("")
+    lines.extend(
+        _render_df(df.head(rows_needed), f"First {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def tail(raw: bytes, n: int = 10) -> bytes:
+    df = _read_df(raw)
+    num_rows = len(df)
+    rows_needed = min(n, num_rows)
+    lines = [f"# Rows: {num_rows}, Columns: {len(df.columns)}", ""]
+    lines.extend(_render_schema(df))
+    lines.append("")
+    lines.extend(
+        _render_df(df.tail(rows_needed), f"Last {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def wc(raw: bytes) -> int:
+    return len(_read_df(raw))
+
+
+def stat(raw: bytes) -> bytes:
+    df = _read_df(raw)
+    lines = [
+        "# HDF5 file",
+        f"rows: {len(df)}",
+        f"columns: {len(df.columns)}",
+        "",
+    ]
+    lines.extend(_render_schema(df))
+    lines.append("")
+    return "\n".join(lines).encode()
+
+
+def grep(raw: bytes, pattern: str, ignore_case: bool = False) -> bytes:
+    flags = re.IGNORECASE if ignore_case else 0
+    regex = re.compile(pattern, flags)
+    df = _read_df(raw)
+    str_cols = df.select_dtypes(include=["object", "string"]).columns
+    if len(str_cols) == 0:
+        return df.head(0).to_csv(index=False).encode()
+    row_mask = pd.Series(False, index=df.index)
+    for col_name in str_cols:
+        row_mask = row_mask | df[col_name].astype(str).str.contains(regex,
+                                                                    na=False)
+    return df[row_mask].to_csv(index=False).encode()
+
+
+def cut(raw: bytes, columns: list[str]) -> bytes:
+    df = _read_df(raw)
+    for col in columns:
+        if col not in df.columns:
+            raise ValueError(f"column not found: {col}")
+    return df[columns].to_csv(index=False).encode()

python/mirage/core/filetype/orc.py

@@ -0,0 +1,171 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import io
+import re
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.orc as orc
+
+_MAX_PREVIEW_ROWS = 20
+
+
+def _open(raw: bytes) -> orc.ORCFile:
+    return orc.ORCFile(io.BytesIO(raw))
+
+
+def _render_schema(schema: pa.Schema) -> list[str]:
+    lines = ["## Schema"]
+    for field in schema:
+        lines.append(f"  {field.name}: {field.type}")
+    return lines
+
+
+def _render_table(table: pa.Table, label: str, count: int) -> list[str]:
+    lines = [f"## {label} ({count} rows)", ""]
+    lines.append(table.to_pandas().to_string(index=False))
+    lines.append("")
+    return lines
+
+
+def cat(raw: bytes, max_rows: int = _MAX_PREVIEW_ROWS) -> bytes:
+    f = _open(raw)
+    schema = f.schema
+    num_rows = f.nrows
+    batches = []
+    collected = 0
+    for i in range(f.nstripes):
+        if collected >= max_rows:
+            break
+        stripe = f.read_stripe(i)
+        batches.append(stripe)
+        collected += stripe.num_rows
+    table = pa.Table.from_batches(batches).slice(0, max_rows)
+    preview_count = min(num_rows, max_rows)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(table, "Preview", preview_count))
+    return "\n".join(lines).encode()
+
+
+def head(raw: bytes, n: int = 10) -> bytes:
+    f = _open(raw)
+    schema = f.schema
+    num_rows = f.nrows
+    rows_needed = min(n, num_rows)
+    batches = []
+    collected = 0
+    for i in range(f.nstripes):
+        if collected >= rows_needed:
+            break
+        stripe = f.read_stripe(i)
+        batches.append(stripe)
+        collected += stripe.num_rows
+    table = pa.Table.from_batches(batches).slice(0, rows_needed)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(table, f"First {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def tail(raw: bytes, n: int = 10) -> bytes:
+    f = _open(raw)
+    schema = f.schema
+    num_rows = f.nrows
+    rows_needed = min(n, num_rows)
+    batches = []
+    collected = 0
+    for i in range(f.nstripes - 1, -1, -1):
+        if collected >= rows_needed:
+            break
+        stripe = f.read_stripe(i)
+        batches.insert(0, stripe)
+        collected += stripe.num_rows
+    combined = pa.Table.from_batches(batches)
+    table = combined.slice(max(0, combined.num_rows - rows_needed),
+                           rows_needed)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(table, f"Last {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def wc(raw: bytes) -> int:
+    f = _open(raw)
+    return f.nrows
+
+
+def stat(raw: bytes) -> bytes:
+    f = _open(raw)
+    schema = f.schema
+    num_rows = f.nrows
+    lines = [
+        "# ORC file",
+        f"rows: {num_rows}",
+        f"columns: {len(schema)}",
+        f"stripes: {f.nstripes}",
+        "",
+    ]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    for i in range(f.nstripes):
+        stripe = f.read_stripe(i)
+        lines.append(f"## Stripe {i}")
+        lines.append(f"  rows: {stripe.num_rows}")
+    lines.append("")
+    return "\n".join(lines).encode()
+
+
+def grep(raw: bytes, pattern: str, ignore_case: bool = False) -> bytes:
+    flags = re.IGNORECASE if ignore_case else 0
+    regex = re.compile(pattern, flags)
+    f = _open(raw)
+    table = f.read()
+    df = table.to_pandas()
+    str_cols = df.select_dtypes(include=["object", "string"]).columns
+    if len(str_cols) == 0:
+        return df.head(0).to_csv(index=False).encode()
+    row_mask = pd.Series(False, index=df.index)
+    for col_name in str_cols:
+        row_mask = row_mask | df[col_name].astype(str).str.contains(regex,
+                                                                    na=False)
+    matched = df[row_mask]
+    return matched.to_csv(index=False).encode()
+
+
+def cut(raw: bytes, columns: list[str]) -> bytes:
+    f = _open(raw)
+    schema_names = f.schema.names
+    for col in columns:
+        if col not in schema_names:
+            raise ValueError(f"column not found: {col}")
+    table = f.read(columns=columns)
+    return table.to_pandas().to_csv(index=False).encode()
+
+
+def file(raw: bytes) -> bytes:
+    f = _open(raw)
+    schema = f.schema
+    cols = ", ".join(f"{field.name}: {field.type}" for field in schema)
+    return (f"orc, {f.nrows} rows, {len(schema)} columns, "
+            f"{f.nstripes} stripes ({cols})").encode()
+
+
+def ls(raw: bytes) -> tuple[int, int]:
+    f = _open(raw)
+    return f.nrows, len(f.schema)

python/mirage/core/filetype/parquet.py

@@ -0,0 +1,174 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import io
+import re
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+_MAX_PREVIEW_ROWS = 20
+
+
+def _open(raw: bytes) -> pq.ParquetFile:
+    return pq.ParquetFile(io.BytesIO(raw))
+
+
+def _render_schema(schema: pa.Schema) -> list[str]:
+    lines = ["## Schema"]
+    for field in schema:
+        lines.append(f"  {field.name}: {field.type}")
+    return lines
+
+
+def _render_table(table: pa.Table, label: str, count: int) -> list[str]:
+    lines = [f"## {label} ({count} rows)", ""]
+    lines.append(table.to_pandas().to_string(index=False))
+    lines.append("")
+    return lines
+
+
+def cat(raw: bytes, max_rows: int = _MAX_PREVIEW_ROWS) -> bytes:
+    pf = _open(raw)
+    schema = pf.schema_arrow
+    num_rows = pf.metadata.num_rows
+    batches = []
+    collected = 0
+    for i in range(pf.metadata.num_row_groups):
+        if collected >= max_rows:
+            break
+        rg = pf.read_row_group(i)
+        batches.append(rg)
+        collected += rg.num_rows
+    table = pa.concat_tables(batches).slice(0, max_rows)
+    preview_count = min(num_rows, max_rows)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(table, "Preview", preview_count))
+    return "\n".join(lines).encode()
+
+
+def head(raw: bytes, n: int = 10) -> bytes:
+    pf = _open(raw)
+    schema = pf.schema_arrow
+    num_rows = pf.metadata.num_rows
+    rows_needed = min(n, num_rows)
+    batches = []
+    collected = 0
+    for i in range(pf.metadata.num_row_groups):
+        if collected >= rows_needed:
+            break
+        rg = pf.read_row_group(i)
+        batches.append(rg)
+        collected += rg.num_rows
+    table = pa.concat_tables(batches).slice(0, rows_needed)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(table, f"First {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def tail(raw: bytes, n: int = 10) -> bytes:
+    pf = _open(raw)
+    schema = pf.schema_arrow
+    num_rows = pf.metadata.num_rows
+    rows_needed = min(n, num_rows)
+    batches = []
+    collected = 0
+    for i in range(pf.metadata.num_row_groups - 1, -1, -1):
+        if collected >= rows_needed:
+            break
+        rg = pf.read_row_group(i)
+        batches.insert(0, rg)
+        collected += rg.num_rows
+    combined = pa.concat_tables(batches)
+    table = combined.slice(max(0, combined.num_rows - rows_needed),
+                           rows_needed)
+    lines = [f"# Rows: {num_rows}, Columns: {len(schema)}", ""]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    lines.extend(_render_table(table, f"Last {rows_needed}", rows_needed))
+    return "\n".join(lines).encode()
+
+
+def wc(raw: bytes) -> int:
+    pf = _open(raw)
+    return pf.metadata.num_rows
+
+
+def stat(raw: bytes) -> bytes:
+    pf = _open(raw)
+    meta = pf.metadata
+    schema = pf.schema_arrow
+    lines = [
+        "# Parquet file",
+        f"rows: {meta.num_rows}",
+        f"columns: {meta.num_columns}",
+        f"row_groups: {meta.num_row_groups}",
+        f"format_version: {meta.format_version}",
+        f"serialized_size: {meta.serialized_size}",
+        "",
+    ]
+    lines.extend(_render_schema(schema))
+    lines.append("")
+    for i in range(meta.num_row_groups):
+        rg = meta.row_group(i)
+        lines.append(f"## Row group {i}")
+        lines.append(f"  rows: {rg.num_rows}")
+        lines.append(f"  total_byte_size: {rg.total_byte_size}")
+    lines.append("")
+    return "\n".join(lines).encode()
+
+
+def grep(raw: bytes, pattern: str, ignore_case: bool = False) -> bytes:
+    flags = re.IGNORECASE if ignore_case else 0
+    regex = re.compile(pattern, flags)
+    pf = _open(raw)
+    table = pf.read()
+    df = table.to_pandas()
+    str_cols = df.select_dtypes(include=["object", "string"]).columns
+    if len(str_cols) == 0:
+        return df.head(0).to_csv(index=False).encode()
+    row_mask = pd.Series(False, index=df.index)
+    for col_name in str_cols:
+        row_mask = row_mask | df[col_name].astype(str).str.contains(regex,
+                                                                    na=False)
+    matched = df[row_mask]
+    return matched.to_csv(index=False).encode()
+
+
+def cut(raw: bytes, columns: list[str]) -> bytes:
+    pf = _open(raw)
+    schema_names = [f.name for f in pf.schema_arrow]
+    for col in columns:
+        if col not in schema_names:
+            raise ValueError(f"column not found: {col}")
+    table = pf.read(columns=columns)
+    return table.to_pandas().to_csv(index=False).encode()
+
+
+def file(raw: bytes) -> bytes:
+    pf = _open(raw)
+    meta = pf.metadata
+    cols = ", ".join(f"{f.name}: {f.type}" for f in pf.schema_arrow)
+    return (f"parquet, {meta.num_rows} rows, {meta.num_columns} columns"
+            f" ({cols})").encode()
+
+
+def ls(raw: bytes) -> tuple[int, int]:
+    pf = _open(raw)
+    return pf.metadata.num_rows, len(pf.schema_arrow)

python/mirage/core/filetype/pdf.py

@@ -0,0 +1,73 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import io
+
+import pypdfium2 as pdfium
+
+_DEFAULT_DPI = 150
+_MAX_PAGES = 20
+
+
+def pages_to_images(
+    raw: bytes,
+    max_pages: int = _MAX_PAGES,
+    dpi: int = _DEFAULT_DPI,
+) -> list[tuple[int, bytes]]:
+    """Convert PDF pages to PNG images.
+
+    Args:
+        raw (bytes): Raw PDF bytes.
+        max_pages (int): Maximum number of pages to render.
+        dpi (int): Rendering resolution.
+
+    Returns:
+        list[tuple[int, bytes]]: List of (1-based page number, PNG bytes).
+    """
+    doc = pdfium.PdfDocument(raw)
+    scale = dpi / 72
+    results: list[tuple[int, bytes]] = []
+    for i in range(min(len(doc), max_pages)):
+        bitmap = doc[i].render(scale=scale)
+        pil_img = bitmap.to_pil()
+        buf = io.BytesIO()
+        pil_img.save(buf, format="PNG")
+        results.append((i + 1, buf.getvalue()))
+    doc.close()
+    return results
+
+
+def cat(raw: bytes, max_pages: int = _MAX_PAGES) -> bytes:
+    """Extract text from PDF for text-mode reads.
+
+    Args:
+        raw (bytes): Raw PDF bytes.
+        max_pages (int): Maximum pages to extract.
+
+    Returns:
+        bytes: UTF-8 encoded text extraction.
+    """
+    doc = pdfium.PdfDocument(raw)
+    total = len(doc)
+    pages_to_read = min(total, max_pages)
+    lines = [f"# PDF: {total} pages"]
+    for i in range(pages_to_read):
+        text_page = doc[i].get_textpage()
+        text = text_page.get_text_range().strip()
+        lines.append(f"\n## Page {i + 1}\n")
+        lines.append(text if text else "(no extractable text)")
+    if total > max_pages:
+        lines.append(f"\n... ({total - max_pages} more pages)")
+    doc.close()
+    return "\n".join(lines).encode()

python/mirage/core/gdocs/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/mirage/core/gdocs/_client.py

@@ -0,0 +1,34 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.core.google._client import DOCS_API_BASE  # noqa: F401
+from mirage.core.google._client import (DRIVE_API_BASE, TOKEN_BUFFER_SECONDS,
+                                        TOKEN_URL, TokenManager, google_get,
+                                        google_get_bytes, google_headers,
+                                        google_post, google_put,
+                                        refresh_access_token)
+
+__all__ = [
+    "DOCS_API_BASE",
+    "DRIVE_API_BASE",
+    "TOKEN_URL",
+    "TOKEN_BUFFER_SECONDS",
+    "TokenManager",
+    "google_get",
+    "google_get_bytes",
+    "google_headers",
+    "google_post",
+    "google_put",
+    "refresh_access_token",
+]