chore: import upstream snapshot with attribution

python/tests/core/s3/__init__.py

@@ -0,0 +1,13 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

python/tests/core/s3/test_index_metadata.py

@@ -0,0 +1,78 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from contextlib import ExitStack
+
+from mirage.accessor.s3 import S3Accessor
+from mirage.cache.index import RAMIndexCacheStore, ResourceType
+from mirage.core.s3.readdir import readdir
+from mirage.core.s3.stat import stat
+from mirage.resource.s3 import S3Config
+from mirage.types import FileType, PathSpec
+from tests.integration.s3_mock import patch_s3_multi
+
+EXPECTED_MODIFIED = "2026-03-31T00:00:00Z"
+
+
+def _accessor():
+    config = S3Config(
+        bucket="test-bucket",
+        region="us-east-1",
+        aws_access_key_id="fake",
+        aws_secret_access_key="fake",
+    )
+    return S3Accessor(config)
+
+
+def test_readdir_stores_remote_time_for_files():
+    store = {"dir/a.txt": b"hello", "dir/sub/b.txt": b"x"}
+    stack = ExitStack()
+    stack.enter_context(patch_s3_multi({"test-bucket": store}))
+    try:
+        accessor = _accessor()
+        cache = RAMIndexCacheStore(ttl=60)
+        scope = PathSpec(resource_path="dir", virtual="/dir", directory="/dir")
+        asyncio.run(readdir(accessor, scope, cache))
+        file_lookup = asyncio.run(cache.get("/dir/a.txt"))
+        assert file_lookup.entry is not None
+        assert file_lookup.entry.resource_type == ResourceType.FILE
+        assert file_lookup.entry.remote_time == EXPECTED_MODIFIED
+        # S3 "folders" are synthetic common-prefixes: no time recorded.
+        dir_lookup = asyncio.run(cache.get("/dir/sub"))
+        assert dir_lookup.entry is not None
+        assert dir_lookup.entry.resource_type == ResourceType.FOLDER
+        assert dir_lookup.entry.remote_time == ""
+    finally:
+        stack.close()
+
+
+def test_stat_returns_modified_from_index():
+    store = {"dir/a.txt": b"hello"}
+    stack = ExitStack()
+    stack.enter_context(patch_s3_multi({"test-bucket": store}))
+    try:
+        accessor = _accessor()
+        cache = RAMIndexCacheStore(ttl=60)
+        scope = PathSpec(resource_path="dir", virtual="/dir", directory="/dir")
+        asyncio.run(readdir(accessor, scope, cache))
+        target = PathSpec(resource_path="dir/a.txt",
+                          virtual="/dir/a.txt",
+                          directory="/dir")
+        result = asyncio.run(stat(accessor, target, index=cache))
+        assert result.modified == EXPECTED_MODIFIED
+        assert result.size == 5
+        assert result.type != FileType.DIRECTORY
+    finally:
+        stack.close()

python/tests/core/s3/test_key_prefix.py

@@ -0,0 +1,155 @@
+import asyncio
+
+from mirage.accessor.s3 import S3Accessor
+from mirage.cache.index.ram import RAMIndexCacheStore
+from mirage.core.s3.copy import copy
+from mirage.core.s3.exists import exists
+from mirage.core.s3.glob import resolve_glob
+from mirage.core.s3.read import read_bytes
+from mirage.core.s3.readdir import readdir
+from mirage.core.s3.rename import rename
+from mirage.core.s3.stat import stat
+from mirage.core.s3.unlink import unlink
+from mirage.core.s3.write import write_bytes
+from mirage.resource.s3 import S3Config
+from mirage.resource.s3.s3 import S3Resource
+from mirage.types import PathSpec
+from tests.integration.s3_mock import patch_s3_multi
+
+PREFIX = "users/abc/"
+BUCKET = "test-bucket"
+
+
+def _config(key_prefix: str | None = None) -> S3Config:
+    return S3Config(
+        bucket=BUCKET,
+        region="us-east-1",
+        aws_access_key_id="fake",
+        aws_secret_access_key="fake",
+        key_prefix=key_prefix,
+    )
+
+
+def _accessor(key_prefix: str | None = None) -> S3Accessor:
+    return S3Accessor(_config(key_prefix))
+
+
+def _path(p: str) -> PathSpec:
+    return PathSpec(virtual=p, directory=p, resource_path=p.strip("/"))
+
+
+def test_normalize_empty_returns_none():
+    assert S3Config(bucket="b", key_prefix="").key_prefix is None
+
+
+def test_normalize_strips_leading_slash():
+    assert S3Config(bucket="b",
+                    key_prefix="/users/abc/").key_prefix == "users/abc/"
+
+
+def test_normalize_adds_trailing_slash():
+    assert S3Config(bucket="b",
+                    key_prefix="users/abc").key_prefix == "users/abc/"
+
+
+def test_write_with_prefix():
+    store = {BUCKET: {}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        asyncio.run(write_bytes(accessor, _path("/a.txt"), b"hello"))
+    assert store[BUCKET].get("users/abc/a.txt") == b"hello"
+
+
+def test_read_baseline_no_prefix():
+    store = {BUCKET: {"a.txt": b"baseline"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(None)
+        data = asyncio.run(read_bytes(accessor, _path("/a.txt")))
+    assert data == b"baseline"
+
+
+def test_readdir_returns_user_facing_paths():
+    store = {BUCKET: {"users/abc/a.txt": b"a", "users/abc/b.txt": b"b"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        index = RAMIndexCacheStore()
+        entries = asyncio.run(readdir(accessor, _path("/"), index=index))
+    for entry in entries:
+        assert "users" not in entry, f"key_prefix leaked into entry: {entry}"
+        assert "abc" not in entry, f"key_prefix leaked into entry: {entry}"
+
+
+def test_resolve_glob_with_prefix():
+    store = {BUCKET: {"users/abc/a.txt": b"a", "users/abc/b.txt": b"b"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        index = RAMIndexCacheStore()
+        glob_path = PathSpec(
+            resource_path="*.txt",
+            virtual="/*.txt",
+            directory="/",
+            pattern="*.txt",
+            resolved=False,
+        )
+        results = asyncio.run(resolve_glob(accessor, [glob_path], index))
+    for r in results:
+        assert "users" not in r.virtual, (
+            f"key_prefix leaked into glob result: {r.virtual}")
+        assert "abc" not in r.virtual, (
+            f"key_prefix leaked into glob result: {r.virtual}")
+
+
+def test_stat_with_prefix():
+    store = {BUCKET: {"users/abc/a.txt": b"data"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        result = asyncio.run(stat(accessor, _path("/a.txt")))
+    assert result.name == "a.txt"
+    assert "users" not in (result.name or "")
+    assert "abc" not in (result.name or "")
+
+
+def test_exists_with_prefix():
+    store = {BUCKET: {"users/abc/a.txt": b"yes"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        found = asyncio.run(exists(accessor, _path("/a.txt")))
+        not_found = asyncio.run(exists(accessor, _path("/missing.txt")))
+    assert found is True
+    assert not_found is False
+
+
+def test_copy_within_prefixed_scope():
+    store = {BUCKET: {"users/abc/a.txt": b"content"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        asyncio.run(copy(accessor, _path("/a.txt"), _path("/b.txt")))
+    assert "users/abc/b.txt" in store[BUCKET]
+    assert store[BUCKET]["users/abc/b.txt"] == b"content"
+    assert "users/abc/a.txt" in store[BUCKET]
+
+
+def test_rename_within_prefixed_scope():
+    store = {BUCKET: {"users/abc/a.txt": b"moved"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        asyncio.run(rename(accessor, _path("/a.txt"), _path("/b.txt")))
+    assert "users/abc/b.txt" in store[BUCKET]
+    assert store[BUCKET]["users/abc/b.txt"] == b"moved"
+    assert "users/abc/a.txt" not in store[BUCKET]
+
+
+def test_unlink_with_prefix():
+    store = {BUCKET: {"users/abc/a.txt": b"gone"}}
+    with patch_s3_multi(store):
+        accessor = _accessor(PREFIX)
+        asyncio.run(unlink(accessor, _path("/a.txt")))
+    assert "users/abc/a.txt" not in store[BUCKET]
+
+
+def test_get_state_includes_key_prefix():
+    config = _config(PREFIX)
+    resource = S3Resource(config)
+    state = resource.get_state()
+    assert state["config"]["key_prefix"] == PREFIX
+    assert state["config"]["key_prefix"] != "<REDACTED>"

python/tests/core/s3/test_key_prefix_threading.py

@@ -0,0 +1,67 @@
+import asyncio
+
+from mirage.accessor.s3 import S3Accessor
+from mirage.cache.index.ram import RAMIndexCacheStore
+from mirage.core.s3.find import find
+from mirage.core.s3.read import read_bytes
+from mirage.core.s3.readdir import readdir
+from mirage.resource.s3 import S3Config
+from mirage.types import PathSpec
+from tests.integration.s3_mock import patch_s3_multi
+
+
+def _make_config(key_prefix: str | None = None) -> S3Config:
+    return S3Config(
+        bucket="test-bucket",
+        region="us-east-1",
+        aws_access_key_id="fake",
+        aws_secret_access_key="fake",
+        key_prefix=key_prefix,
+    )
+
+
+def test_readdir_no_prefix_backward_compat():
+    store = {"test-bucket": {"dir/a.txt": b"a", "dir/b.txt": b"b"}}
+    with patch_s3_multi(store):
+        config = _make_config(key_prefix=None)
+        accessor = S3Accessor(config)
+        path = PathSpec(resource_path="dir", virtual="/dir", directory="/dir")
+        index = RAMIndexCacheStore()
+        entries = asyncio.run(readdir(accessor, path, index=index))
+    assert any(e.endswith("/a.txt") for e in entries)
+    assert any(e.endswith("/b.txt") for e in entries)
+
+
+def test_readdir_strips_key_prefix_from_entries():
+    store = {"test-bucket": {"prod/dir/a.txt": b"a", "prod/dir/b.txt": b"b"}}
+    with patch_s3_multi(store):
+        config = _make_config(key_prefix="prod")
+        accessor = S3Accessor(config)
+        path = PathSpec(resource_path="dir", virtual="/dir", directory="/dir")
+        index = RAMIndexCacheStore()
+        entries = asyncio.run(readdir(accessor, path, index=index))
+    for e in entries:
+        assert "prod" not in e, f"key_prefix leaked into entry: {e}"
+
+
+def test_read_bytes_with_key_prefix():
+    store = {"test-bucket": {"prod/hello.txt": b"hello"}}
+    with patch_s3_multi(store):
+        config = _make_config(key_prefix="prod")
+        accessor = S3Accessor(config)
+        path = PathSpec(resource_path="hello.txt",
+                        virtual="/hello.txt",
+                        directory="/hello.txt")
+        data = asyncio.run(read_bytes(accessor, path))
+    assert data == b"hello"
+
+
+def test_find_strips_key_prefix_from_results():
+    store = {"test-bucket": {"prod/dir/a.txt": b"a", "prod/dir/b.txt": b"b"}}
+    with patch_s3_multi(store):
+        config = _make_config(key_prefix="prod")
+        accessor = S3Accessor(config)
+        path = PathSpec(resource_path="dir", virtual="/dir", directory="/dir")
+        results = asyncio.run(find(accessor, path))
+    for r in results:
+        assert "prod" not in r, f"key_prefix leaked into find result: {r}"

python/tests/core/s3/test_readdir_paths.py

@@ -0,0 +1,62 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+
+import pytest
+
+from mirage.resource.ram import RAMResource
+from mirage.types import MountMode
+from mirage.workspace import Workspace
+
+
+@pytest.fixture
+def ws():
+    mem = RAMResource()
+    w = Workspace(resources={"/mnt/data": (mem, MountMode.WRITE)})
+    asyncio.run(w.execute("mkdir /mnt/data/dir"))
+    asyncio.run(w.execute("echo -n a > /mnt/data/dir/a.txt"))
+    asyncio.run(w.execute("echo -n b > /mnt/data/dir/b.txt"))
+    asyncio.run(w.execute("echo -n c > /mnt/data/c.csv"))
+    return w
+
+
+def test_readdir_returns_leading_slash(ws):
+    entries = asyncio.run(ws.ops.readdir("/mnt/data/dir"))
+    for e in entries:
+        assert e.startswith("/"), f"readdir entry missing leading /: {e}"
+
+
+def test_readdir_returns_full_virtual_paths(ws):
+    entries = asyncio.run(ws.ops.readdir("/mnt/data/dir"))
+    assert "/mnt/data/dir/a.txt" in entries
+    assert "/mnt/data/dir/b.txt" in entries
+
+
+def test_readdir_root(ws):
+    entries = asyncio.run(ws.ops.readdir("/mnt/data"))
+    names = [e.rsplit("/", 1)[-1] for e in entries]
+    assert "dir" in names
+    assert "c.csv" in names
+
+
+def test_readdir_glob_expansion(ws):
+
+    async def _run():
+        io = await ws.execute("echo /mnt/data/dir/*.txt")
+        return await io.stdout_str()
+
+    out = asyncio.run(_run()).strip()
+    assert "/mnt/data/dir/a.txt" in out
+    assert "/mnt/data/dir/b.txt" in out

python/tests/core/s3/test_stat_directory.py

@@ -0,0 +1,54 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from contextlib import ExitStack
+
+from mirage.accessor.s3 import S3Accessor
+from mirage.core.s3.stat import stat
+from mirage.resource.s3 import S3Config
+from mirage.types import FileType, PathSpec
+from tests.integration.s3_mock import patch_s3_multi
+
+
+def _accessor() -> S3Accessor:
+    return S3Accessor(
+        S3Config(
+            bucket="test-bucket",
+            region="us-east-1",
+            aws_access_key_id="fake",
+            aws_secret_access_key="fake",
+        ))
+
+
+def test_trailing_slash_prefers_directory_over_coexisting_object():
+    store = {"csv": b"i am a file", "csv/1.txt": b"child"}
+    stack = ExitStack()
+    stack.enter_context(patch_s3_multi({"test-bucket": store}))
+    try:
+        accessor = _accessor()
+        file_stat = asyncio.run(
+            stat(accessor,
+                 PathSpec(resource_path="csv", virtual="/csv", directory="/"),
+                 index=None))
+        assert file_stat.type != FileType.DIRECTORY
+        dir_stat = asyncio.run(
+            stat(accessor,
+                 PathSpec(resource_path="csv",
+                          virtual="/csv/",
+                          directory="/csv/"),
+                 index=None))
+        assert dir_stat.type == FileType.DIRECTORY
+    finally:
+        stack.close()

python/tests/core/s3/test_stat_fingerprint.py

@@ -0,0 +1,44 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from contextlib import ExitStack
+
+from mirage.accessor.s3 import S3Accessor
+from mirage.core.s3.stat import stat
+from mirage.resource.s3 import S3Config
+from mirage.types import PathSpec
+from tests.integration.s3_mock import patch_s3_multi
+
+
+def test_s3_stat_returns_fingerprint_from_etag():
+    store = {"foo.txt": b"hello"}
+    stack = ExitStack()
+    stack.enter_context(patch_s3_multi({"test-bucket": store}))
+    try:
+        config = S3Config(
+            bucket="test-bucket",
+            region="us-east-1",
+            aws_access_key_id="fake",
+            aws_secret_access_key="fake",
+        )
+        accessor = S3Accessor(config)
+        scope = PathSpec(resource_path="foo.txt",
+                         virtual="/foo.txt",
+                         directory="/")
+        result = asyncio.run(stat(accessor, scope, index=None))
+        assert result.fingerprint is not None
+        assert result.fingerprint == result.extra.get("etag")
+    finally:
+        stack.close()