chore: import upstream snapshot with attribution

python/mirage/core/nextcloud/constants.py

@@ -0,0 +1,3 @@
+SCOPE_WARN = 500
+SCOPE_ERROR = 5000
+DEFAULT_CHUNK_SIZE = 8192

python/mirage/core/nextcloud/copy.py

@@ -0,0 +1,22 @@
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def copy(accessor: NextcloudAccessor, src: PathSpec,
+               dst: PathSpec) -> None:
+    if isinstance(src, str):
+        src = PathSpec.from_str_path(src)
+    if isinstance(dst, str):
+        dst = PathSpec.from_str_path(dst)
+    src_key = src.mount_path.lstrip("/")
+    dst_key = dst.mount_path.lstrip("/")
+    op = accessor.operator()
+    try:
+        await op.copy(src_key, dst_key)
+    except NotFound as exc:
+        raise enoent(src) from exc
+    await invalidate_after_write(dst)

python/mirage/core/nextcloud/create.py

@@ -0,0 +1,12 @@
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+async def create(accessor: NextcloudAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    key = path.mount_path.lstrip("/")
+    op = accessor.operator()
+    await op.write(key, b"")
+    await invalidate_after_write(path)

python/mirage/core/nextcloud/du.py

@@ -0,0 +1,63 @@
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.core.nextcloud.stat import stat
+from mirage.types import FileType, PathSpec
+
+
+async def du(accessor: NextcloudAccessor, path: PathSpec) -> int:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    try:
+        info = await stat(accessor, path)
+    except FileNotFoundError:
+        info = None
+    if info is not None and info.type != FileType.DIRECTORY:
+        return info.size or 0
+    target = path.mount_path
+    pfx = target.strip("/")
+    scan_path = pfx + "/" if pfx else "/"
+    op = accessor.operator()
+    total = 0
+    try:
+        async for entry in await op.scan(scan_path):
+            if entry.path.endswith("/"):
+                continue
+            meta = entry.metadata
+            if meta is not None:
+                total += int(meta.content_length or 0)
+    except NotFound:
+        return 0
+    return total
+
+
+async def du_all(accessor: NextcloudAccessor,
+                 path: PathSpec) -> list[tuple[str, int]]:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    try:
+        info = await stat(accessor, path)
+    except FileNotFoundError:
+        info = None
+    if info is not None and info.type != FileType.DIRECTORY:
+        return []
+    target = path.mount_path
+    pfx = target.strip("/")
+    scan_path = pfx + "/" if pfx else "/"
+    op = accessor.operator()
+    results: list[tuple[str, int]] = []
+    total = 0
+    try:
+        async for entry in await op.scan(scan_path):
+            rel = entry.path
+            if not rel or rel.endswith("/"):
+                continue
+            meta = entry.metadata
+            sz = int(meta.content_length or 0) if meta is not None else 0
+            results.append(("/" + rel.lstrip("/"), sz))
+            total += sz
+    except NotFound:
+        pass
+    results.sort()
+    results.append((target, total))
+    return results

python/mirage/core/nextcloud/exists.py

@@ -0,0 +1,11 @@
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.core.nextcloud.stat import stat
+from mirage.types import PathSpec
+
+
+async def exists(accessor: NextcloudAccessor, path: PathSpec) -> bool:
+    try:
+        await stat(accessor, path)
+        return True
+    except (FileNotFoundError, ValueError):
+        return False

python/mirage/core/nextcloud/find.py

@@ -0,0 +1,122 @@
+from opendal.exceptions import NotFound
+from opendal.types import EntryMode
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
+                                               emit_start_path, keep,
+                                               start_basename)
+from mirage.types import PathSpec
+
+
+async def find(
+    accessor: NextcloudAccessor,
+    path: PathSpec,
+    name: str | None = None,
+    type: str | None = None,
+    min_size: int | None = None,
+    max_size: int | None = None,
+    maxdepth: int | None = None,
+    name_exclude: str | None = None,
+    or_names: list[str] | None = None,
+    mtime_min: float | None = None,
+    mtime_max: float | None = None,
+    iname: str | None = None,
+    path_pattern: str | None = None,
+    mindepth: int | None = None,
+    empty: bool = False,
+    tree: PredNode | None = None,
+) -> list[str]:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    start_name = start_basename(path)
+    target = path.mount_path
+    pfx = target.strip("/")
+    scan_path = pfx + "/" if pfx else "/"
+    base = "/" + pfx if pfx else "/"
+    base_depth = 0 if base == "/" else base.count("/")
+
+    op = accessor.operator()
+    results: list[str] = []
+    seen_dirs: set[str] = set()
+    saw_descendant = False
+    dir_exists = False
+    tree = tree if tree is not None else build_tree(name=name,
+                                                    iname=iname,
+                                                    path_pattern=path_pattern,
+                                                    type=type,
+                                                    name_exclude=name_exclude,
+                                                    or_names=or_names)
+    try:
+        async for entry in await op.scan(scan_path):
+            rel = entry.path
+            if not rel:
+                continue
+            meta = entry.metadata
+            is_dir = (rel.endswith("/")
+                      or getattr(meta, "mode", None) == EntryMode.Dir)
+            entry_path = "/" + rel.rstrip("/").lstrip("/")
+            if entry_path == base:
+                dir_exists = True
+                continue
+            saw_descendant = True
+            kind = "d" if is_dir else "f"
+            content_length = getattr(meta, "content_length", 0) or 0
+            last_modified = getattr(meta, "last_modified", None)
+
+            file_entries: list[tuple[str, str]] = [(entry_path, kind)]
+            if not is_dir:
+                parent = entry_path.rsplit("/", 1)[0] or "/"
+                while parent and parent != base and parent != "/":
+                    if parent not in seen_dirs:
+                        seen_dirs.add(parent)
+                        file_entries.append((parent, "d"))
+                    parent = parent.rsplit("/", 1)[0] or "/"
+
+            for ep, k in file_entries:
+                en = ep.rsplit("/", 1)[-1]
+                depth = ep.count("/") - base_depth
+                if maxdepth is not None and depth > maxdepth:
+                    continue
+                fe = FindEntry(
+                    key=ep,
+                    name=en,
+                    kind=k,
+                    depth=depth,
+                    is_empty=False if k == "d" else content_length == 0)
+                if not keep(fe, tree, mindepth):
+                    continue
+
+                if min_size is not None or max_size is not None:
+                    # Directories count as size 0 for -size (deliberate GNU
+                    # divergence).
+                    size = content_length if k == "f" else 0
+                    if min_size is not None and size < min_size:
+                        continue
+                    if max_size is not None and size > max_size:
+                        continue
+
+                if mtime_min is not None or mtime_max is not None:
+                    if last_modified is None:
+                        continue
+                    mt = last_modified.timestamp()
+                    if mtime_min is not None and mt < mtime_min:
+                        continue
+                    if mtime_max is not None and mt > mtime_max:
+                        continue
+
+                results.append(ep)
+    except NotFound:
+        return []
+    if saw_descendant or dir_exists:
+        emit_start_path(results,
+                        base,
+                        start_name,
+                        kind="d",
+                        is_empty=False,
+                        exists=True,
+                        tree=tree,
+                        maxdepth=maxdepth,
+                        mindepth=mindepth,
+                        min_size=min_size,
+                        max_size=max_size)
+    return sorted(set(results))

python/mirage/core/nextcloud/glob.py

@@ -0,0 +1,15 @@
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.nextcloud.constants import SCOPE_ERROR
+from mirage.core.nextcloud.readdir import readdir
+from mirage.types import PathSpec
+from mirage.utils.glob_walk import resolve_glob_with
+
+
+async def resolve_glob(
+    accessor: NextcloudAccessor,
+    paths: list[PathSpec],
+    index: IndexCacheStore,
+) -> list[PathSpec]:
+    return await resolve_glob_with(readdir, accessor, paths, index,
+                                   SCOPE_ERROR)

python/mirage/core/nextcloud/mkdir.py

@@ -0,0 +1,15 @@
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+async def mkdir(accessor: NextcloudAccessor,
+                path: PathSpec,
+                parents: bool = False) -> None:
+    # opendal create_dir creates missing parents; parents is implicit.
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    key = path.mount_path.strip("/") + "/"
+    op = accessor.operator()
+    await op.create_dir(key)
+    await invalidate_after_write(path)

python/mirage/core/nextcloud/read.py

@@ -0,0 +1,35 @@
+import time
+
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.observe.context import record
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def read_bytes(accessor: NextcloudAccessor,
+                     path: PathSpec,
+                     index: IndexCacheStore = None,
+                     offset: int = 0,
+                     size: int | None = None) -> bytes:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.lstrip("/")
+    op = accessor.operator()
+    start_ms = int(time.monotonic() * 1000)
+    try:
+        if offset or size is not None:
+            async with await op.open(key, "rb") as f:
+                if offset:
+                    await f.seek(offset)
+                data = await f.read(size
+                                    ) if size is not None else await f.read()
+        else:
+            data = bytes(await op.read(key))
+    except NotFound as exc:
+        raise enoent(path) from exc
+    record("read", raw, "nextcloud", len(data), start_ms)
+    return data

python/mirage/core/nextcloud/readdir.py

@@ -0,0 +1,85 @@
+import logging
+
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.index import IndexCacheStore, IndexEntry, ResourceType
+from mirage.core.nextcloud.constants import SCOPE_ERROR
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent, enotdir
+from mirage.utils.key_prefix import mount_prefix_of
+
+logger = logging.getLogger(__name__)
+
+
+async def readdir(accessor: NextcloudAccessor, path: PathSpec,
+                  index: IndexCacheStore) -> list[str]:
+    if isinstance(path, str):
+        path = PathSpec(virtual=path,
+                        directory=path,
+                        resource_path=path.strip("/"))
+    prefix = mount_prefix_of(path.virtual, path.resource_path)
+    target = path.directory if path.pattern else path.virtual
+    if prefix and target.startswith(prefix):
+        rest = target[len(prefix):]
+        if prefix.endswith("/") or rest == "" or rest.startswith("/"):
+            target = rest or "/"
+    virtual_key = (prefix + target if prefix else target).rstrip("/") or "/"
+    listing = await index.list_dir(virtual_key)
+    if listing.entries is not None:
+        return listing.entries
+    list_path = target.strip("/")
+    list_path = list_path + "/" if list_path else "/"
+    op = accessor.operator()
+    names: list[str] = []
+    dir_keys: set[str] = set()
+    sizes: dict[str, int | None] = {}
+    times: dict[str, str] = {}
+    try:
+        async for entry in await op.list(list_path):
+            relative = entry.path
+            if not relative or relative == list_path:
+                continue
+            is_dir = relative.endswith("/")
+            base = "/" + relative.rstrip("/")
+            names.append(base)
+            meta = entry.metadata
+            if meta and meta.last_modified:
+                times[base] = meta.last_modified.isoformat()
+            if is_dir:
+                dir_keys.add(base)
+            else:
+                sizes[base] = meta.content_length if meta else None
+    except NotFound as exc:
+        raise enoent(path) from exc
+    # WebDAV PROPFIND on a file returns the file itself; POSIX readdir of a
+    # non-directory raises ENOTDIR instead.
+    target_key = "/" + target.strip("/")
+    if names == [target_key] and target_key not in dir_keys:
+        raise enotdir(path)
+    names = sorted(names)
+    if len(names) > SCOPE_ERROR:
+        logger.warning(
+            "nextcloud readdir: %s returned %d entries (limit %d)",
+            virtual_key,
+            len(names),
+            SCOPE_ERROR,
+        )
+    virtual_entries = sorted((prefix + e if prefix else e) for e in names)
+    index_entries: list[tuple[str, IndexEntry]] = []
+    for e in names:
+        name = e.rsplit("/", 1)[-1]
+        if e in dir_keys:
+            entry_obj = IndexEntry(id=e,
+                                   name=name,
+                                   resource_type=ResourceType.FOLDER,
+                                   remote_time=times.get(e, ""))
+        else:
+            entry_obj = IndexEntry(id=e,
+                                   name=name,
+                                   resource_type=ResourceType.FILE,
+                                   size=sizes.get(e),
+                                   remote_time=times.get(e, ""))
+        index_entries.append((name, entry_obj))
+    await index.set_dir(virtual_key, index_entries)
+    return virtual_entries

python/mirage/core/nextcloud/rename.py

@@ -0,0 +1,24 @@
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import (invalidate_after_unlink,
+                                  invalidate_after_write)
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def rename(accessor: NextcloudAccessor, src: PathSpec,
+                 dst: PathSpec) -> None:
+    if isinstance(src, str):
+        src = PathSpec.from_str_path(src)
+    if isinstance(dst, str):
+        dst = PathSpec.from_str_path(dst)
+    src_key = src.mount_path.lstrip("/")
+    dst_key = dst.mount_path.lstrip("/")
+    op = accessor.operator()
+    try:
+        await op.rename(src_key, dst_key)
+    except NotFound as exc:
+        raise enoent(src) from exc
+    await invalidate_after_write(dst)
+    await invalidate_after_unlink(src)

python/mirage/core/nextcloud/rm.py

@@ -0,0 +1,19 @@
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def rm_r(accessor: NextcloudAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.strip("/") + "/"
+    op = accessor.operator()
+    try:
+        await op.remove_all(key)
+    except NotFound as exc:
+        raise enoent(path) from exc
+    await invalidate_after_unlink(path)

python/mirage/core/nextcloud/rmdir.py

@@ -0,0 +1,19 @@
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def rmdir(accessor: NextcloudAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.strip("/") + "/"
+    op = accessor.operator()
+    try:
+        await op.delete(key)
+    except NotFound as exc:
+        raise enoent(path) from exc
+    await invalidate_after_unlink(path)

python/mirage/core/nextcloud/stat.py

@@ -0,0 +1,67 @@
+from opendal.exceptions import NotFound
+from opendal.types import EntryMode
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.index import IndexCacheStore, ResourceType
+from mirage.types import FileStat, FileType, PathSpec
+from mirage.utils.errors import enoent
+from mirage.utils.filetype import guess_type
+from mirage.utils.key_prefix import mount_prefix_of
+
+
+async def stat(accessor: NextcloudAccessor,
+               path: PathSpec,
+               index: IndexCacheStore = None) -> FileStat:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    original_prefix = mount_prefix_of(path.virtual, path.resource_path)
+    raw = path.virtual
+    if original_prefix and raw.startswith(original_prefix):
+        raw = raw[len(original_prefix):] or "/"
+    stripped = raw.strip("/")
+    if not stripped:
+        return FileStat(name="/", type=FileType.DIRECTORY)
+    if index is not None:
+        virtual_key = (original_prefix + "/" +
+                       stripped if original_prefix else "/" + stripped)
+        lookup = await index.get(virtual_key)
+        if lookup.entry is not None:
+            entry = lookup.entry
+            if entry.resource_type == ResourceType.FOLDER:
+                return FileStat(name=entry.name,
+                                type=FileType.DIRECTORY,
+                                modified=entry.remote_time or None)
+            return FileStat(name=entry.name,
+                            size=entry.size,
+                            modified=entry.remote_time or None,
+                            type=guess_type(entry.name))
+        parent = virtual_key.rsplit("/", 1)[0] or "/"
+        parent_listing = await index.list_dir(parent)
+        if parent_listing.entries is not None:
+            raise enoent(path)
+    op = accessor.operator()
+    key = stripped
+    try:
+        md = await op.stat(key)
+    except NotFound:
+        md = None
+    if md is not None and md.mode != EntryMode.Dir:
+        modified = md.last_modified.isoformat() if md.last_modified else None
+        return FileStat(
+            name=stripped.rsplit("/", 1)[-1],
+            size=md.content_length,
+            modified=modified,
+            type=guess_type(raw),
+            fingerprint=md.etag,
+            extra={"etag": md.etag} if md.etag else {},
+        )
+    try:
+        md_dir = await op.stat(key + "/")
+        if md_dir and md_dir.mode == EntryMode.Dir:
+            return FileStat(
+                name=stripped.rsplit("/", 1)[-1] or "/",
+                type=FileType.DIRECTORY,
+            )
+    except NotFound:
+        pass
+    raise enoent(path)

python/mirage/core/nextcloud/stream.py

@@ -0,0 +1,56 @@
+import time
+from collections.abc import AsyncIterator
+
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.index import IndexCacheStore
+from mirage.core.nextcloud.constants import DEFAULT_CHUNK_SIZE
+from mirage.observe.context import record, record_stream
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def range_read(accessor: NextcloudAccessor, path: PathSpec, start: int,
+                     end: int) -> bytes:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.lstrip("/")
+    op = accessor.operator()
+    start_ms = int(time.monotonic() * 1000)
+    try:
+        async with await op.open(key, "rb") as f:
+            if start:
+                await f.seek(start)
+            data = await f.read(end - start)
+    except NotFound as exc:
+        raise enoent(path) from exc
+    record("read", raw, "nextcloud", len(data), start_ms)
+    return data
+
+
+async def read_stream(
+    accessor: NextcloudAccessor,
+    path: PathSpec,
+    index: IndexCacheStore = None,
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+) -> AsyncIterator[bytes]:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.lstrip("/")
+    op = accessor.operator()
+    rec = record_stream("read", raw, "nextcloud")
+    try:
+        async with await op.open(key, "rb") as f:
+            while True:
+                chunk = await f.read(chunk_size)
+                if not chunk:
+                    break
+                chunk_bytes = bytes(chunk)
+                if rec is not None:
+                    rec.bytes += len(chunk_bytes)
+                yield chunk_bytes
+    except NotFound as exc:
+        raise enoent(path) from exc

python/mirage/core/nextcloud/truncate.py

@@ -0,0 +1,20 @@
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.types import PathSpec
+
+
+async def truncate(accessor: NextcloudAccessor, path: PathSpec,
+                   length: int) -> None:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    key = path.mount_path.lstrip("/")
+    op = accessor.operator()
+    try:
+        data = bytes(await op.read(key))
+    except NotFound:
+        data = b""
+    result = data[:length].ljust(length, b"\0")
+    await op.write(key, result)
+    await invalidate_after_write(path)

python/mirage/core/nextcloud/unlink.py

@@ -0,0 +1,24 @@
+import time
+
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_unlink
+from mirage.observe.context import record
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def unlink(accessor: NextcloudAccessor, path: PathSpec) -> None:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.lstrip("/")
+    op = accessor.operator()
+    start_ms = int(time.monotonic() * 1000)
+    try:
+        await op.delete(key)
+    except NotFound as exc:
+        raise enoent(path) from exc
+    record("unlink", path.virtual, "nextcloud", 0, start_ms)
+    await invalidate_after_unlink(path)

python/mirage/core/nextcloud/write.py

@@ -0,0 +1,28 @@
+import time
+
+from opendal.exceptions import NotFound
+
+from mirage.accessor.nextcloud import NextcloudAccessor
+from mirage.cache.context import invalidate_after_write
+from mirage.cache.index import IndexCacheStore
+from mirage.observe.context import record
+from mirage.types import PathSpec
+from mirage.utils.errors import enoent
+
+
+async def write_bytes(accessor: NextcloudAccessor,
+                      path: PathSpec,
+                      data: bytes,
+                      index: IndexCacheStore = None) -> None:
+    if isinstance(path, str):
+        path = PathSpec.from_str_path(path)
+    raw = path.mount_path
+    key = raw.lstrip("/")
+    op = accessor.operator()
+    start_ms = int(time.monotonic() * 1000)
+    try:
+        await op.write(key, data)
+    except NotFound as exc:
+        raise enoent(path) from exc
+    record("write", path.virtual, "nextcloud", len(data), start_ms)
+    await invalidate_after_write(path)