chore: import upstream snapshot with attribution

python/tests/core/hf_buckets/conftest.py

@@ -0,0 +1,174 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+import pytest
+from opendal.exceptions import NotFound
+from opendal.types import EntryMode
+
+from mirage.accessor.hf_buckets import HfBucketsAccessor, HfBucketsConfig
+
+
+@dataclass
+class _FakeMetadata:
+    content_length: int = 0
+    mode: EntryMode = EntryMode.File
+    etag: str | None = None
+    last_modified: datetime | None = None
+    content_type: str | None = None
+
+
+@dataclass
+class _FakeEntry:
+    path: str
+    metadata: _FakeMetadata
+
+    @property
+    def name(self) -> str:
+        return self.path.rstrip("/").rsplit("/", 1)[-1]
+
+
+@dataclass
+class _FakeFile:
+    data: bytes
+    pos: int = 0
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    async def seek(self, offset: int) -> None:
+        self.pos = offset
+
+    async def read(self, size: int | None = None) -> bytes:
+        if size is None:
+            out = self.data[self.pos:]
+            self.pos = len(self.data)
+        else:
+            out = self.data[self.pos:self.pos + size]
+            self.pos += len(out)
+        return out
+
+    async def tell(self) -> int:
+        return self.pos
+
+
+@dataclass
+class FakeAsyncOperator:
+    files: dict[str, bytes] = field(default_factory=dict)
+    metas: dict[str, _FakeMetadata] = field(default_factory=dict)
+
+    def __post_init__(self):
+        for k, data in self.files.items():
+            self.metas.setdefault(
+                k,
+                _FakeMetadata(
+                    content_length=len(data),
+                    mode=EntryMode.File,
+                    etag=f"etag-{k}",
+                    last_modified=datetime(2026, 1, 1, tzinfo=timezone.utc),
+                ),
+            )
+
+    async def read(self, key: str) -> bytes:
+        if key not in self.files:
+            raise NotFound("path not found", key)
+        return self.files[key]
+
+    async def open(self, key: str, mode: str = "rb"):
+        if key not in self.files:
+            raise NotFound("path not found", key)
+        return _FakeFile(data=self.files[key])
+
+    async def stat(self, key: str) -> _FakeMetadata:
+        k = key.rstrip("/")
+        if key.endswith("/"):
+            if any(f.startswith(k + "/") for f in self.files):
+                return _FakeMetadata(content_length=0, mode=EntryMode.Dir)
+            raise NotFound("path not found", key)
+        if k in self.files:
+            return self.metas[k]
+        raise NotFound("path not found", key)
+
+    async def write(self, key: str, data: bytes) -> None:
+        self.files[key] = bytes(data)
+        self.metas[key] = _FakeMetadata(
+            content_length=len(data),
+            mode=EntryMode.File,
+            etag=f"etag-{key}",
+        )
+
+    async def delete(self, key: str) -> None:
+        self.files.pop(key, None)
+        self.metas.pop(key, None)
+
+    async def list(self, path: str):
+        pfx = path.lstrip("/")
+        seen_dirs: set[str] = set()
+        entries: list[_FakeEntry] = []
+        for f in self.files:
+            if not f.startswith(pfx):
+                continue
+            rest = f[len(pfx):]
+            if "/" in rest:
+                subdir = rest.split("/", 1)[0]
+                dkey = pfx + subdir + "/"
+                if dkey not in seen_dirs:
+                    seen_dirs.add(dkey)
+                    entries.append(
+                        _FakeEntry(
+                            path=dkey,
+                            metadata=_FakeMetadata(mode=EntryMode.Dir),
+                        ))
+            else:
+                entries.append(_FakeEntry(path=f, metadata=self.metas[f]))
+
+        async def _iter():
+            for e in entries:
+                yield e
+
+        return _iter()
+
+    async def scan(self, path: str):
+        pfx = path.lstrip("/")
+        entries = [
+            _FakeEntry(path=f, metadata=self.metas[f]) for f in self.files
+            if f.startswith(pfx)
+        ]
+
+        async def _iter():
+            for e in entries:
+                yield e
+
+        return _iter()
+
+
+def make_accessor(files: dict[str, bytes] | None = None,
+                  *,
+                  key_prefix: str | None = None) -> HfBucketsAccessor:
+    cfg = HfBucketsConfig(bucket="o/b", token="t", key_prefix=key_prefix)
+    acc = HfBucketsAccessor(cfg)
+    fake = FakeAsyncOperator(files=dict(files or {}))
+    acc._fake = fake
+    acc.operator = lambda: fake
+    return acc
+
+
+@pytest.fixture
+def make_acc():
+    return make_accessor

python/tests/core/hf_buckets/test_du.py

@@ -0,0 +1,59 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.du import du, du_all
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_du_sums_file_sizes_recursive(make_acc):
+    acc = make_acc({
+        "data/a.json": b"12345",
+        "data/sub/b.json": b"67",
+        "other.txt": b"x",
+    })
+    total = await du(acc, PathSpec.from_str_path("/data"))
+    assert total == 7
+
+
+@pytest.mark.asyncio
+async def test_du_all_returns_per_file_with_total(make_acc):
+    acc = make_acc({
+        "data/a.json": b"12345",
+        "data/b.json": b"67",
+    })
+    out = await du_all(acc, PathSpec.from_str_path("/data"))
+    assert out[-1] == ("/data", 7)
+    files = sorted(out[:-1])
+    assert files == [("/data/a.json", 5), ("/data/b.json", 2)]
+
+
+@pytest.mark.asyncio
+async def test_du_missing_returns_zero(make_acc):
+    acc = make_acc({})
+    assert await du(acc, PathSpec.from_str_path("/nope")) == 0
+
+
+@pytest.mark.asyncio
+async def test_du_of_file_returns_its_own_size(make_acc):
+    acc = make_acc({"data/a.json": b"12345"})
+    assert await du(acc, PathSpec.from_str_path("/data/a.json")) == 5
+
+
+@pytest.mark.asyncio
+async def test_du_all_of_file_is_empty(make_acc):
+    acc = make_acc({"data/a.json": b"12345"})
+    assert await du_all(acc, PathSpec.from_str_path("/data/a.json")) == []

python/tests/core/hf_buckets/test_exists.py

@@ -0,0 +1,30 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.exists import exists
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_exists_true_for_file(make_acc):
+    acc = make_acc({"a.txt": b"x"})
+    assert await exists(acc, PathSpec.from_str_path("/a.txt")) is True
+
+
+@pytest.mark.asyncio
+async def test_exists_false_for_missing(make_acc):
+    acc = make_acc({})
+    assert await exists(acc, PathSpec.from_str_path("/missing.txt")) is False

python/tests/core/hf_buckets/test_find.py

@@ -0,0 +1,81 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.find import find
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_find_root_returns_sorted_entries(make_acc):
+    acc = make_acc({
+        "a.json": b"a",
+        "b.json": b"b",
+        "data/c.json": b"c",
+    })
+    out = await find(acc, PathSpec.from_str_path("/"))
+    assert out == ["/", "/a.json", "/b.json", "/data", "/data/c.json"]
+
+
+@pytest.mark.asyncio
+async def test_find_subdir_scopes_results(make_acc):
+    acc = make_acc({
+        "data/a.json": b"a",
+        "data/sub/b.json": b"b",
+        "other.txt": b"o",
+    })
+    out = await find(acc, PathSpec.from_str_path("/data"))
+    assert out == ["/data", "/data/a.json", "/data/sub", "/data/sub/b.json"]
+
+
+@pytest.mark.asyncio
+async def test_find_missing_returns_empty(make_acc):
+    acc = make_acc({})
+    out = await find(acc, PathSpec.from_str_path("/nope"))
+    assert out == []
+
+
+@pytest.mark.asyncio
+async def test_find_name_filter(make_acc):
+    acc = make_acc({
+        "a.json": b"a",
+        "b.txt": b"b",
+        "data/c.json": b"c",
+    })
+    out = await find(acc, PathSpec.from_str_path("/"), name="*.json")
+    assert out == ["/a.json", "/data/c.json"]
+
+
+@pytest.mark.asyncio
+async def test_find_type_filter(make_acc):
+    acc = make_acc({
+        "a.json": b"a",
+        "data/c.json": b"c",
+    })
+    files = await find(acc, PathSpec.from_str_path("/"), type="f")
+    dirs = await find(acc, PathSpec.from_str_path("/"), type="d")
+    assert files == ["/a.json", "/data/c.json"]
+    assert dirs == ["/", "/data"]
+
+
+@pytest.mark.asyncio
+async def test_find_maxdepth(make_acc):
+    acc = make_acc({
+        "a.json": b"a",
+        "data/c.json": b"c",
+        "data/sub/d.json": b"d",
+    })
+    out = await find(acc, PathSpec.from_str_path("/"), maxdepth=1)
+    assert out == ["/", "/a.json", "/data"]

python/tests/core/hf_buckets/test_glob.py

@@ -0,0 +1,57 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.cache.index import RAMIndexCacheStore
+from mirage.core.hf_buckets.glob import resolve_glob
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_resolve_glob_passes_through_resolved_paths(make_acc):
+    acc = make_acc({})
+    index = RAMIndexCacheStore(ttl=60)
+    p = PathSpec.from_str_path("/data/a.txt")
+    out = await resolve_glob(acc, [p], index)
+    assert out == [p]
+
+
+@pytest.mark.asyncio
+async def test_resolve_glob_expands_basename_pattern(make_acc):
+    acc = make_acc({
+        "data/a.txt": b"a",
+        "data/b.json": b"b",
+        "data/sub/c.txt": b"c",
+    })
+    index = RAMIndexCacheStore(ttl=60)
+    pattern = PathSpec(
+        resource_path="data/*.txt",
+        virtual="/data/*.txt",
+        directory="/data/",
+        pattern="*.txt",
+        resolved=False,
+    )
+    out = await resolve_glob(acc, [pattern], index)
+    paths = sorted(p.virtual for p in out)
+    assert paths == ["/data/a.txt"]
+
+
+@pytest.mark.asyncio
+async def test_resolve_glob_string_input_passthrough(make_acc):
+    acc = make_acc({})
+    index = RAMIndexCacheStore(ttl=60)
+    out = await resolve_glob(acc, ["/data/a.txt"], index)
+    assert len(out) == 1
+    assert out[0].virtual == "/data/a.txt"

python/tests/core/hf_buckets/test_read.py

@@ -0,0 +1,46 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.read import read_bytes
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_read_bytes_whole_file(make_acc):
+    acc = make_acc({"greet.txt": b"hello world"})
+    out = await read_bytes(acc, PathSpec.from_str_path("/greet.txt"))
+    assert out == b"hello world"
+
+
+@pytest.mark.asyncio
+async def test_read_bytes_offset_size_returns_slice(make_acc):
+    acc = make_acc({"x": b"abcdef"})
+    out = await read_bytes(acc, PathSpec.from_str_path("/x"), offset=2, size=4)
+    assert out == b"cdef"
+
+
+@pytest.mark.asyncio
+async def test_read_bytes_missing_raises_filenotfound(make_acc):
+    acc = make_acc({})
+    with pytest.raises(FileNotFoundError):
+        await read_bytes(acc, PathSpec.from_str_path("/nope"))
+
+
+@pytest.mark.asyncio
+async def test_read_bytes_offset_only(make_acc):
+    acc = make_acc({"x": b"abcdef"})
+    out = await read_bytes(acc, PathSpec.from_str_path("/x"), offset=3)
+    assert out == b"def"

python/tests/core/hf_buckets/test_readdir.py

@@ -0,0 +1,51 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.cache.index import RAMIndexCacheStore
+from mirage.core.hf_buckets.readdir import readdir
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_readdir_root_returns_children(make_acc):
+    acc = make_acc({"hello.txt": b"x", "data/file.txt": b"y"})
+    entries = await readdir(acc, PathSpec.from_str_path("/"),
+                            RAMIndexCacheStore(ttl=60))
+    assert "/data" in entries
+    assert "/hello.txt" in entries
+
+
+@pytest.mark.asyncio
+async def test_readdir_subdir(make_acc):
+    acc = make_acc({
+        "data/a.txt": b"a",
+        "data/sub/b.txt": b"b",
+        "other.txt": b"o"
+    })
+    entries = await readdir(acc, PathSpec.from_str_path("/data"),
+                            RAMIndexCacheStore(ttl=60))
+    assert sorted(entries) == ["/data/a.txt", "/data/sub"]
+
+
+@pytest.mark.asyncio
+async def test_readdir_populates_index_cache(make_acc):
+    acc = make_acc({"f.txt": b"hello"})
+    cache = RAMIndexCacheStore(ttl=60)
+    await readdir(acc, PathSpec.from_str_path("/"), cache)
+    lookup = await cache.get("/f.txt")
+    assert lookup.entry is not None
+    assert lookup.entry.size == 5
+    assert lookup.entry.resource_type == "file"

python/tests/core/hf_buckets/test_stat.py

@@ -0,0 +1,50 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.stat import stat
+from mirage.types import FileType, PathSpec
+
+
+@pytest.mark.asyncio
+async def test_stat_file_returns_size_and_etag(make_acc):
+    acc = make_acc({"data/file.txt": b"abcde"})
+    s = await stat(acc, PathSpec.from_str_path("/data/file.txt"))
+    assert s.name == "file.txt"
+    assert s.size == 5
+    assert s.fingerprint == "etag-data/file.txt"
+    assert s.type != FileType.DIRECTORY
+
+
+@pytest.mark.asyncio
+async def test_stat_directory_via_dir_probe(make_acc):
+    acc = make_acc({"data/file.txt": b"x"})
+    s = await stat(acc, PathSpec.from_str_path("/data"))
+    assert s.type == FileType.DIRECTORY
+    assert s.name == "data"
+
+
+@pytest.mark.asyncio
+async def test_stat_missing_raises_filenotfound(make_acc):
+    acc = make_acc({})
+    with pytest.raises(FileNotFoundError):
+        await stat(acc, PathSpec.from_str_path("/missing.txt"))
+
+
+@pytest.mark.asyncio
+async def test_stat_root_is_directory(make_acc):
+    acc = make_acc({})
+    s = await stat(acc, PathSpec.from_str_path("/"))
+    assert s.type == FileType.DIRECTORY

python/tests/core/hf_buckets/test_stream.py

@@ -0,0 +1,45 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.stream import range_read, read_stream
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_range_read_returns_slice(make_acc):
+    acc = make_acc({"x": b"abcdef"})
+    out = await range_read(acc, PathSpec.from_str_path("/x"), 1, 4)
+    assert out == b"bcd"
+
+
+@pytest.mark.asyncio
+async def test_read_stream_yields_chunks(make_acc):
+    acc = make_acc({"x": b"abcdefgh"})
+    chunks = []
+    async for c in read_stream(acc, PathSpec.from_str_path("/x"),
+                               chunk_size=3):
+        chunks.append(c)
+    assert b"".join(chunks) == b"abcdefgh"
+    assert len(chunks) >= 2
+
+
+@pytest.mark.asyncio
+async def test_read_stream_handles_empty_file(make_acc):
+    acc = make_acc({"empty": b""})
+    chunks = [
+        c async for c in read_stream(acc, PathSpec.from_str_path("/empty"))
+    ]
+    assert b"".join(chunks) == b""

python/tests/core/hf_buckets/test_write_ops.py

@@ -0,0 +1,49 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import pytest
+
+from mirage.core.hf_buckets.create import create
+from mirage.core.hf_buckets.unlink import unlink
+from mirage.core.hf_buckets.write import write_bytes
+from mirage.types import PathSpec
+
+
+@pytest.mark.asyncio
+async def test_write_bytes_uploads(make_acc):
+    acc = make_acc({})
+    await write_bytes(acc, PathSpec.from_str_path("/hello.txt"), b"hi there")
+    assert acc._fake.files == {"hello.txt": b"hi there"}
+
+
+@pytest.mark.asyncio
+async def test_unlink_deletes_file(make_acc):
+    acc = make_acc({"delete-me.txt": b"x"})
+    await unlink(acc, PathSpec.from_str_path("/delete-me.txt"))
+    assert "delete-me.txt" not in acc._fake.files
+
+
+@pytest.mark.asyncio
+async def test_unlink_refuses_directory(make_acc):
+    acc = make_acc({"some-dir/child.txt": b"x"})
+    with pytest.raises(IsADirectoryError):
+        await unlink(acc, PathSpec.from_str_path("/some-dir"))
+    assert "some-dir/child.txt" in acc._fake.files
+
+
+@pytest.mark.asyncio
+async def test_create_writes_empty_file(make_acc):
+    acc = make_acc({})
+    await create(acc, PathSpec.from_str_path("/touched.txt"))
+    assert acc._fake.files.get("touched.txt") == b""