chore: import upstream snapshot with attribution
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
import posixpath
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import unquote
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.accessor.databricks_volume import DatabricksVolumeAccessor
|
||||
from mirage.cache.index import RAMIndexCacheStore
|
||||
from mirage.core.databricks_volume.path import backend_path
|
||||
from mirage.resource.databricks_volume import DatabricksVolumeConfig
|
||||
|
||||
|
||||
class NotFoundError(Exception):
|
||||
status_code = 404
|
||||
|
||||
|
||||
class ToThreadRecorder:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = []
|
||||
|
||||
async def __call__(self, fn, *args, **kwargs):
|
||||
self.calls.append((fn, args, kwargs))
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
|
||||
class FakeDownload:
|
||||
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.contents = BytesIO(data)
|
||||
|
||||
|
||||
class FakeFiles:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.downloads: dict[str, bytes] = {}
|
||||
self.metadata: dict[str, object] = {}
|
||||
self.directory_metadata: set[str] = set()
|
||||
self.directories: dict[str, list[object]] = {}
|
||||
self.metadata_errors: dict[str, Exception] = {}
|
||||
self.directory_metadata_errors: dict[str, Exception] = {}
|
||||
self.download_calls: list[str] = []
|
||||
self.get_metadata_calls: list[str] = []
|
||||
self.get_directory_metadata_calls: list[str] = []
|
||||
self.list_directory_calls: list[str] = []
|
||||
self.upload_calls: list[tuple[str, bytes, bool]] = []
|
||||
self.delete_calls: list[str] = []
|
||||
self.create_directory_calls: list[str] = []
|
||||
self.delete_directory_calls: list[str] = []
|
||||
|
||||
def download(self, path: str) -> FakeDownload:
|
||||
self.download_calls.append(path)
|
||||
if path not in self.downloads:
|
||||
raise NotFoundError(path)
|
||||
return FakeDownload(self.downloads[path])
|
||||
|
||||
def get_metadata(self, path: str) -> object:
|
||||
self.get_metadata_calls.append(path)
|
||||
if path in self.metadata_errors:
|
||||
raise self.metadata_errors[path]
|
||||
if path not in self.metadata:
|
||||
raise NotFoundError(path)
|
||||
return self.metadata[path]
|
||||
|
||||
def get_directory_metadata(self, path: str) -> None:
|
||||
self.get_directory_metadata_calls.append(path)
|
||||
if path in self.directory_metadata_errors:
|
||||
raise self.directory_metadata_errors[path]
|
||||
if path not in self.directory_metadata:
|
||||
raise NotFoundError(path)
|
||||
|
||||
def list_directory_contents(self, path: str) -> list[object]:
|
||||
self.list_directory_calls.append(path)
|
||||
if path not in self.directories:
|
||||
raise NotFoundError(path)
|
||||
return self.directories[path]
|
||||
|
||||
def create_directory(self, path: str) -> None:
|
||||
self.create_directory_calls.append(path)
|
||||
cur = ""
|
||||
for part in path.strip("/").split("/"):
|
||||
cur = cur + "/" + part
|
||||
if cur in self.directory_metadata:
|
||||
continue
|
||||
self.directory_metadata.add(cur)
|
||||
self.metadata[cur] = SimpleNamespace(is_directory=True)
|
||||
self.directories.setdefault(cur, [])
|
||||
parent = posixpath.dirname(cur) or "/"
|
||||
self._upsert_directory_entry(
|
||||
parent, SimpleNamespace(path=cur, is_directory=True))
|
||||
|
||||
def delete_directory(self, path: str) -> None:
|
||||
self.delete_directory_calls.append(path)
|
||||
if path not in self.directory_metadata:
|
||||
raise NotFoundError(path)
|
||||
if self.directories.get(path):
|
||||
raise OSError(f"directory not empty: {path}")
|
||||
self.directory_metadata.discard(path)
|
||||
self.metadata.pop(path, None)
|
||||
self.directories.pop(path, None)
|
||||
parent = posixpath.dirname(path.rstrip("/")) or "/"
|
||||
self.directories[parent] = [
|
||||
entry for entry in self.directories.get(parent, [])
|
||||
if getattr(entry, "path", None) != path
|
||||
]
|
||||
|
||||
def upload(self, path: str, contents, overwrite: bool = False) -> None:
|
||||
data = contents.read()
|
||||
self.upload_calls.append((path, data, overwrite))
|
||||
parent = posixpath.dirname(path.rstrip("/")) or "/"
|
||||
if parent not in self.directory_metadata:
|
||||
if parent in self.metadata:
|
||||
raise NotADirectoryError(parent)
|
||||
raise NotFoundError(parent)
|
||||
if path in self.directory_metadata:
|
||||
raise IsADirectoryError(path)
|
||||
self.downloads[path] = data
|
||||
self.metadata[path] = file_metadata(len(data))
|
||||
self._upsert_directory_entry(parent, file_entry(path, len(data)))
|
||||
|
||||
def delete(self, path: str) -> None:
|
||||
self.delete_calls.append(path)
|
||||
if path in self.directory_metadata:
|
||||
raise IsADirectoryError(path)
|
||||
if path not in self.metadata and path not in self.downloads:
|
||||
raise NotFoundError(path)
|
||||
self.metadata.pop(path, None)
|
||||
self.downloads.pop(path, None)
|
||||
parent = posixpath.dirname(path.rstrip("/")) or "/"
|
||||
self.directories[parent] = [
|
||||
entry for entry in self.directories.get(parent, [])
|
||||
if getattr(entry, "path", None) != path
|
||||
]
|
||||
|
||||
def _upsert_directory_entry(self, parent: str, entry: object) -> None:
|
||||
entries = [
|
||||
existing for existing in self.directories.get(parent, [])
|
||||
if getattr(existing, "path", None) != getattr(entry, "path", None)
|
||||
]
|
||||
entries.append(entry)
|
||||
self.directories[parent] = sorted(
|
||||
entries, key=lambda item: getattr(item, "path", ""))
|
||||
|
||||
|
||||
def _apply_range_header(data: bytes, range_header: str) -> bytes:
|
||||
if not range_header.startswith("bytes="):
|
||||
raise ValueError(f"unsupported range header: {range_header}")
|
||||
start_text, end_text = range_header.removeprefix("bytes=").split("-", 1)
|
||||
start = int(start_text) if start_text else 0
|
||||
end = int(end_text) + 1 if end_text else None
|
||||
return data[start:end]
|
||||
|
||||
|
||||
class FakeApiClient:
|
||||
|
||||
def __init__(self, files: FakeFiles) -> None:
|
||||
self.files = files
|
||||
self.do_calls: list[dict[str, object]] = []
|
||||
|
||||
def do(
|
||||
self,
|
||||
method: str,
|
||||
path: str | None = None,
|
||||
url: str | None = None,
|
||||
query: dict | None = None,
|
||||
headers: dict | None = None,
|
||||
body: dict | None = None,
|
||||
raw: bool = False,
|
||||
files: object = None,
|
||||
data: object = None,
|
||||
auth: object = None,
|
||||
response_headers: list[str] | None = None,
|
||||
) -> dict:
|
||||
call = {
|
||||
"method": method,
|
||||
"path": path,
|
||||
"url": url,
|
||||
"query": query,
|
||||
"headers": headers or {},
|
||||
"body": body,
|
||||
"raw": raw,
|
||||
"files": files,
|
||||
"data": data,
|
||||
"auth": auth,
|
||||
"response_headers": response_headers,
|
||||
}
|
||||
self.do_calls.append(call)
|
||||
if method != "GET" or path is None:
|
||||
raise ValueError(f"unsupported fake API call: {method} {path}")
|
||||
remote_path = unquote(path.removeprefix("/api/2.0/fs/files"))
|
||||
if remote_path not in self.files.downloads:
|
||||
raise NotFoundError(remote_path)
|
||||
payload = self.files.downloads[remote_path]
|
||||
range_header = (headers or {}).get("Range")
|
||||
if range_header is not None:
|
||||
payload = _apply_range_header(payload, range_header)
|
||||
return {
|
||||
"contents": BytesIO(payload),
|
||||
"content-length": str(len(payload)),
|
||||
"accept-ranges": "bytes",
|
||||
}
|
||||
|
||||
|
||||
class FakeClient:
|
||||
|
||||
def __init__(self, files: FakeFiles) -> None:
|
||||
self.files = files
|
||||
self.api_client = FakeApiClient(files)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def databricks_config() -> DatabricksVolumeConfig:
|
||||
return DatabricksVolumeConfig(
|
||||
catalog="main",
|
||||
schema="default",
|
||||
volume="agent_files",
|
||||
root_path="/root",
|
||||
token="secret",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def remote_root(databricks_config: DatabricksVolumeConfig) -> str:
|
||||
return backend_path(databricks_config, "/")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def files() -> FakeFiles:
|
||||
return FakeFiles()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def accessor(
|
||||
databricks_config: DatabricksVolumeConfig,
|
||||
files: FakeFiles,
|
||||
) -> DatabricksVolumeAccessor:
|
||||
return DatabricksVolumeAccessor(databricks_config, FakeClient(files))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def index() -> RAMIndexCacheStore:
|
||||
return RAMIndexCacheStore(ttl=600)
|
||||
|
||||
|
||||
def file_metadata(size: int = 0, modified: str | None = None) -> object:
|
||||
return SimpleNamespace(
|
||||
content_length=size,
|
||||
content_type=None,
|
||||
last_modified=modified,
|
||||
)
|
||||
|
||||
|
||||
def directory_entry(path: str, modified: int | None = None) -> object:
|
||||
return SimpleNamespace(path=path,
|
||||
is_directory=True,
|
||||
file_size=None,
|
||||
last_modified=modified)
|
||||
|
||||
|
||||
def file_entry(path: str,
|
||||
size: int = 0,
|
||||
modified: int | None = None) -> object:
|
||||
return SimpleNamespace(path=path,
|
||||
is_directory=False,
|
||||
file_size=size,
|
||||
last_modified=modified)
|
||||
@@ -0,0 +1,148 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.copy import copy
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _path(path: str) -> PathSpec:
|
||||
return PathSpec.from_str_path(path, mount_key(path, "/dbx"))
|
||||
|
||||
|
||||
def _seed_directory(files, path: str) -> None:
|
||||
files.directory_metadata.add(path)
|
||||
files.directories.setdefault(path, [])
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
if parent and parent != path:
|
||||
files.directories.setdefault(parent, []).append(
|
||||
SimpleNamespace(path=path, is_directory=True, file_size=None))
|
||||
|
||||
|
||||
def _seed_file(files, path: str, data: bytes) -> None:
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
files.downloads[path] = data
|
||||
files.metadata[path] = SimpleNamespace(is_directory=False,
|
||||
file_size=len(data))
|
||||
files.directories.setdefault(parent, []).append(
|
||||
SimpleNamespace(path=path, is_directory=False, file_size=len(data)))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_file_preserves_bytes(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/src.txt", b"hello")
|
||||
|
||||
await copy(accessor, _path("/dbx/src.txt"), _path("/dbx/dst.txt"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/dst.txt"] == b"hello"
|
||||
assert files.downloads[f"{remote_root}/src.txt"] == b"hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_missing_source_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await copy(accessor, _path("/dbx/missing.txt"), _path("/dbx/dst.txt"),
|
||||
index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_missing_parent_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/src.txt", b"hi")
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await copy(accessor, _path("/dbx/src.txt"),
|
||||
_path("/dbx/missing/dst.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_directory_without_recursive_fails(accessor, files,
|
||||
remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
|
||||
with pytest.raises(IsADirectoryError):
|
||||
await copy(accessor, _path("/dbx/d"), _path("/dbx/d2"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_same_path_is_noop(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/src.txt", b"hello")
|
||||
|
||||
await copy(accessor, _path("/dbx/src.txt"), _path("/dbx/src.txt"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/src.txt"] == b"hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_same_missing_path_fails(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await copy(accessor, _path("/dbx/missing.txt"),
|
||||
_path("/dbx/missing.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_same_dir_without_recursive_fails(accessor, files,
|
||||
remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
|
||||
with pytest.raises(IsADirectoryError):
|
||||
await copy(accessor, _path("/dbx/d"), _path("/dbx/d"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_recursive_tree(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
_seed_file(files, f"{remote_root}/d/a.txt", b"aaa")
|
||||
|
||||
await copy(accessor,
|
||||
_path("/dbx/d"),
|
||||
_path("/dbx/d2"),
|
||||
index,
|
||||
recursive=True)
|
||||
|
||||
assert f"{remote_root}/d2" in files.directory_metadata
|
||||
assert files.downloads[f"{remote_root}/d2/a.txt"] == b"aaa"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_copy_recursive_into_own_subtree_fails(accessor, files,
|
||||
remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
_seed_file(files, f"{remote_root}/d/a.txt", b"aaa")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await copy(accessor,
|
||||
_path("/dbx/d"),
|
||||
_path("/dbx/d/d"),
|
||||
index,
|
||||
recursive=True)
|
||||
|
||||
assert files.create_directory_calls == []
|
||||
assert files.upload_calls == []
|
||||
assert files.downloads[f"{remote_root}/d/a.txt"] == b"aaa"
|
||||
@@ -0,0 +1,23 @@
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.exists import exists
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
from .conftest import file_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exists_true_for_file(accessor, files, remote_root):
|
||||
files.metadata[f"{remote_root}/reports/latest.md"] = file_metadata(size=6)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
assert await exists(accessor, path) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exists_false_for_missing_path(accessor):
|
||||
path = PathSpec.from_str_path("/volume/missing.md",
|
||||
mount_key("/volume/missing.md", "/volume"))
|
||||
assert await exists(accessor, path) is False
|
||||
@@ -0,0 +1,46 @@
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.glob import resolve_glob
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
from .conftest import file_entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_file_path(accessor, index):
|
||||
scope = PathSpec.from_str_path("/volume/readme.md",
|
||||
mount_key("/volume/readme.md", "/volume"))
|
||||
result = await resolve_glob(accessor, [scope], index)
|
||||
assert result == [scope]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_glob_pattern(accessor, files, index, remote_root):
|
||||
files.directories[f"{remote_root}/src"] = [
|
||||
file_entry(f"{remote_root}/src/main.py"),
|
||||
file_entry(f"{remote_root}/src/util.py"),
|
||||
file_entry(f"{remote_root}/src/data.json"),
|
||||
]
|
||||
scope = PathSpec(
|
||||
resource_path=mount_key("/volume/src/*.py", "/volume"),
|
||||
virtual="/volume/src/*.py",
|
||||
directory="/volume/src",
|
||||
pattern="*.py",
|
||||
resolved=False,
|
||||
)
|
||||
result = await resolve_glob(accessor, [scope], index)
|
||||
originals = sorted(path.virtual for path in result)
|
||||
assert originals == ["/volume/src/main.py", "/volume/src/util.py"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_directory_path(accessor, index):
|
||||
scope = PathSpec(
|
||||
resource_path=mount_key("/volume/src", "/volume"),
|
||||
virtual="/volume/src",
|
||||
directory="/volume/src",
|
||||
resolved=False,
|
||||
)
|
||||
result = await resolve_glob(accessor, [scope], index)
|
||||
assert result == [scope]
|
||||
@@ -0,0 +1,29 @@
|
||||
import pytest
|
||||
|
||||
from mirage.commands.builtin.databricks_volume.head import head
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
async def collect_bytes(source) -> bytes:
|
||||
return b"".join([chunk async for chunk in source])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_head_bytes_mode_uses_single_small_range_request(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
source, _io = await head(accessor, [path], c="3")
|
||||
|
||||
assert await collect_bytes(source) == b"abc"
|
||||
assert files.download_calls == []
|
||||
assert len(accessor.client.api_client.do_calls) == 1
|
||||
assert accessor.client.api_client.do_calls[0]["headers"]["Range"] == (
|
||||
"bytes=0-2")
|
||||
@@ -0,0 +1,79 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.mkdir import mkdir
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _path(path: str) -> PathSpec:
|
||||
return PathSpec.from_str_path(path, mount_key(path, "/dbx"))
|
||||
|
||||
|
||||
def _seed_directory(files, path: str) -> None:
|
||||
files.directory_metadata.add(path)
|
||||
files.directories.setdefault(path, [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mkdir_creates_directory(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
await mkdir(accessor, _path("/dbx/newdir"), index)
|
||||
|
||||
assert f"{remote_root}/newdir" in files.create_directory_calls
|
||||
assert f"{remote_root}/newdir" in files.directory_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mkdir_parent_missing_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await mkdir(accessor, _path("/dbx/a/b"), index)
|
||||
assert files.create_directory_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mkdir_parents_creates_chain(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
await mkdir(accessor, _path("/dbx/a/b/c"), index, parents=True)
|
||||
|
||||
assert f"{remote_root}/a/b/c" in files.directory_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mkdir_existing_target_fails(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/exists")
|
||||
|
||||
with pytest.raises(FileExistsError):
|
||||
await mkdir(accessor, _path("/dbx/exists"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mkdir_parent_is_file_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
files.metadata[f"{remote_root}/file.txt"] = SimpleNamespace(
|
||||
is_directory=False, file_size=3)
|
||||
|
||||
with pytest.raises(NotADirectoryError):
|
||||
await mkdir(accessor, _path("/dbx/file.txt/sub"), index)
|
||||
@@ -0,0 +1,60 @@
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from mirage.core.databricks_volume.path import backend_path, virtual_path
|
||||
from mirage.resource.databricks_volume import DatabricksVolumeConfig
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def test_backend_path_uses_volume_root_and_strips_mount_prefix(
|
||||
databricks_config):
|
||||
path = PathSpec(
|
||||
resource_path=mount_key("/volume/reports/latest.md", "/volume"),
|
||||
virtual="/volume/reports/latest.md",
|
||||
directory="/volume/reports",
|
||||
)
|
||||
assert backend_path(
|
||||
databricks_config,
|
||||
path) == ("/Volumes/main/default/agent_files/root/reports/latest.md")
|
||||
|
||||
|
||||
def test_backend_path_allows_normalized_path_inside_root(databricks_config):
|
||||
path = PathSpec(
|
||||
resource_path=mount_key("/volume/reports/../latest.md", "/volume"),
|
||||
virtual="/volume/reports/../latest.md",
|
||||
directory="/volume/reports",
|
||||
)
|
||||
assert backend_path(
|
||||
databricks_config,
|
||||
path) == ("/Volumes/main/default/agent_files/root/latest.md")
|
||||
|
||||
|
||||
def test_backend_path_rejects_escape_above_configured_root(databricks_config):
|
||||
path = PathSpec(
|
||||
resource_path=mount_key(
|
||||
"/volume/../../other_schema/other_volume/secret.txt", "/volume"),
|
||||
virtual="/volume/../../other_schema/other_volume/secret.txt",
|
||||
directory="/volume",
|
||||
)
|
||||
with pytest.raises(ValueError, match="escapes Databricks volume root"):
|
||||
backend_path(databricks_config, path)
|
||||
|
||||
|
||||
def test_config_rejects_parent_segments_in_root_path():
|
||||
with pytest.raises(ValidationError):
|
||||
DatabricksVolumeConfig(
|
||||
catalog="main",
|
||||
schema="default",
|
||||
volume="agent_files",
|
||||
root_path="/root/../other",
|
||||
)
|
||||
|
||||
|
||||
def test_virtual_path_rejects_backend_outside_root(databricks_config):
|
||||
with pytest.raises(ValueError, match="outside Databricks volume root"):
|
||||
virtual_path(
|
||||
databricks_config,
|
||||
"/Volumes/main/default/other_volume/secret.txt",
|
||||
"/volume",
|
||||
)
|
||||
@@ -0,0 +1,115 @@
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.read import read_bytes
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
from .conftest import ToThreadRecorder
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file(accessor, files, remote_root):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"hello"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
result = await read_bytes(accessor, path)
|
||||
assert result == b"hello"
|
||||
assert files.download_calls == [f"{remote_root}/reports/latest.md"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file_not_found(accessor):
|
||||
path = PathSpec.from_str_path("/volume/missing.md",
|
||||
mount_key("/volume/missing.md", "/volume"))
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await read_bytes(accessor, path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_slice(accessor, files, remote_root):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
result = await read_bytes(accessor, path, offset=1, size=3)
|
||||
assert result == b"bcd"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file_runs_blocking_download_off_event_loop(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
monkeypatch,
|
||||
):
|
||||
to_thread = ToThreadRecorder()
|
||||
monkeypatch.setattr(asyncio, "to_thread", to_thread)
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"hello"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
result = await read_bytes(accessor, path)
|
||||
|
||||
assert result == b"hello"
|
||||
assert len(to_thread.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_slice_uses_databricks_range_request(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
result = await read_bytes(accessor, path, offset=1, size=3)
|
||||
|
||||
assert result == b"bcd"
|
||||
assert files.download_calls == []
|
||||
assert accessor.client.api_client.do_calls[0]["headers"]["Range"] == (
|
||||
"bytes=1-3")
|
||||
assert accessor.client.api_client.do_calls[0]["raw"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_from_offset_uses_open_ended_range(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
result = await read_bytes(accessor, path, offset=3)
|
||||
|
||||
assert result == b"def"
|
||||
assert files.download_calls == []
|
||||
assert accessor.client.api_client.do_calls[0]["headers"]["Range"] == (
|
||||
"bytes=3-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_zero_size_returns_empty_without_network(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
result = await read_bytes(accessor, path, size=0)
|
||||
|
||||
assert result == b""
|
||||
assert files.download_calls == []
|
||||
assert accessor.client.api_client.do_calls == []
|
||||
@@ -0,0 +1,101 @@
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.readdir import readdir
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
from .conftest import ToThreadRecorder, directory_entry, file_entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readdir_returns_full_virtual_paths(
|
||||
accessor,
|
||||
files,
|
||||
index,
|
||||
remote_root,
|
||||
):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md", size=6),
|
||||
directory_entry(f"{remote_root}/reports/archive"),
|
||||
]
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
result = await readdir(accessor, path, index)
|
||||
assert result == [
|
||||
"/volume/reports/archive",
|
||||
"/volume/reports/latest.md",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readdir_uses_cached_listing(accessor, files, index,
|
||||
remote_root):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md", size=6),
|
||||
]
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
assert await readdir(accessor, path,
|
||||
index) == ["/volume/reports/latest.md"]
|
||||
files.directories[f"{remote_root}/reports"] = []
|
||||
assert await readdir(accessor, path,
|
||||
index) == ["/volume/reports/latest.md"]
|
||||
assert files.list_directory_calls == [f"{remote_root}/reports"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readdir_populates_index_with_size_and_modified(
|
||||
accessor,
|
||||
files,
|
||||
index,
|
||||
remote_root,
|
||||
):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md",
|
||||
size=6,
|
||||
modified=1_700_000_000_000),
|
||||
directory_entry(f"{remote_root}/reports/archive"),
|
||||
]
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
await readdir(accessor, path, index)
|
||||
file_lookup = await index.get("/volume/reports/latest.md")
|
||||
assert file_lookup.entry is not None
|
||||
assert file_lookup.entry.resource_type == "file"
|
||||
assert file_lookup.entry.size == 6
|
||||
assert file_lookup.entry.remote_time == "2023-11-14T22:13:20+00:00"
|
||||
dir_lookup = await index.get("/volume/reports/archive")
|
||||
assert dir_lookup.entry is not None
|
||||
assert dir_lookup.entry.resource_type == "folder"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readdir_missing_directory_raises(accessor, index):
|
||||
path = PathSpec.from_str_path("/volume/missing",
|
||||
mount_key("/volume/missing", "/volume"))
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await readdir(accessor, path, index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readdir_runs_blocking_list_off_event_loop(
|
||||
accessor,
|
||||
files,
|
||||
index,
|
||||
remote_root,
|
||||
monkeypatch,
|
||||
):
|
||||
to_thread = ToThreadRecorder()
|
||||
monkeypatch.setattr(asyncio, "to_thread", to_thread)
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md", size=6),
|
||||
]
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
|
||||
result = await readdir(accessor, path, index)
|
||||
|
||||
assert result == ["/volume/reports/latest.md"]
|
||||
assert len(to_thread.calls) == 1
|
||||
@@ -0,0 +1,116 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.rename import rename
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _path(path: str) -> PathSpec:
|
||||
return PathSpec.from_str_path(path, mount_key(path, "/dbx"))
|
||||
|
||||
|
||||
def _seed_directory(files, path: str) -> None:
|
||||
files.directory_metadata.add(path)
|
||||
files.directories.setdefault(path, [])
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
if parent and parent != path:
|
||||
files.directories.setdefault(parent, []).append(
|
||||
SimpleNamespace(path=path, is_directory=True, file_size=None))
|
||||
|
||||
|
||||
def _seed_file(files, path: str, data: bytes) -> None:
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
files.downloads[path] = data
|
||||
files.metadata[path] = SimpleNamespace(is_directory=False,
|
||||
file_size=len(data))
|
||||
files.directories.setdefault(parent, []).append(
|
||||
SimpleNamespace(path=path, is_directory=False, file_size=len(data)))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_file_moves_bytes(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/src.txt", b"data")
|
||||
|
||||
await rename(accessor, _path("/dbx/src.txt"), _path("/dbx/dst.txt"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/dst.txt"] == b"data"
|
||||
assert f"{remote_root}/src.txt" not in files.downloads
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_missing_source_fails(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await rename(accessor, _path("/dbx/missing.txt"),
|
||||
_path("/dbx/dst.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_same_path_is_noop(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/src.txt", b"data")
|
||||
|
||||
await rename(accessor, _path("/dbx/src.txt"), _path("/dbx/src.txt"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/src.txt"] == b"data"
|
||||
assert f"{remote_root}/src.txt" not in files.delete_calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_same_missing_path_fails(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await rename(accessor, _path("/dbx/missing.txt"),
|
||||
_path("/dbx/missing.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_directory_moves_tree(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
_seed_file(files, f"{remote_root}/d/a.txt", b"aaa")
|
||||
|
||||
await rename(accessor, _path("/dbx/d"), _path("/dbx/d2"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/d2/a.txt"] == b"aaa"
|
||||
assert f"{remote_root}/d" not in files.directory_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_into_own_subtree_fails(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
_seed_file(files, f"{remote_root}/d/a.txt", b"aaa")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await rename(accessor, _path("/dbx/d"), _path("/dbx/d/d"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/d/a.txt"] == b"aaa"
|
||||
assert f"{remote_root}/d" in files.directory_metadata
|
||||
assert files.create_directory_calls == []
|
||||
assert files.upload_calls == []
|
||||
assert files.delete_calls == []
|
||||
assert files.delete_directory_calls == []
|
||||
@@ -0,0 +1,81 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.rm import rm_recursive
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _path(path: str) -> PathSpec:
|
||||
return PathSpec.from_str_path(path, mount_key(path, "/dbx"))
|
||||
|
||||
|
||||
def _seed_directory(files, path: str) -> None:
|
||||
files.directory_metadata.add(path)
|
||||
files.directories.setdefault(path, [])
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
if parent and parent != path:
|
||||
files.directories.setdefault(parent, []).append(
|
||||
SimpleNamespace(path=path, is_directory=True, file_size=None))
|
||||
|
||||
|
||||
def _seed_file(files, path: str, data: bytes = b"x") -> None:
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
files.downloads[path] = data
|
||||
files.metadata[path] = SimpleNamespace(is_directory=False,
|
||||
file_size=len(data))
|
||||
files.directories.setdefault(parent, []).append(
|
||||
SimpleNamespace(path=path, is_directory=False, file_size=len(data)))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rm_recursive_file(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/a.txt")
|
||||
|
||||
removed = await rm_recursive(accessor, _path("/dbx/a.txt"), index)
|
||||
|
||||
assert removed == ["/a.txt"]
|
||||
assert f"{remote_root}/a.txt" not in files.downloads
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rm_recursive_nested_tree(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/d")
|
||||
_seed_file(files, f"{remote_root}/d/a.txt")
|
||||
_seed_directory(files, f"{remote_root}/d/sub")
|
||||
_seed_file(files, f"{remote_root}/d/sub/b.txt")
|
||||
|
||||
removed = await rm_recursive(accessor, _path("/dbx/d"), index)
|
||||
|
||||
assert f"{remote_root}/d/a.txt" in files.delete_calls
|
||||
assert f"{remote_root}/d/sub/b.txt" in files.delete_calls
|
||||
assert f"{remote_root}/d/sub" in files.delete_directory_calls
|
||||
assert f"{remote_root}/d" in files.delete_directory_calls
|
||||
assert "/d" in removed
|
||||
assert f"{remote_root}/d" not in files.directory_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rm_recursive_missing_raises(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await rm_recursive(accessor, _path("/dbx/missing"), index)
|
||||
@@ -0,0 +1,79 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.rmdir import rmdir
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _path(path: str) -> PathSpec:
|
||||
return PathSpec.from_str_path(path, mount_key(path, "/dbx"))
|
||||
|
||||
|
||||
def _seed_directory(files, path: str) -> None:
|
||||
files.directory_metadata.add(path)
|
||||
files.directories.setdefault(path, [])
|
||||
|
||||
|
||||
def _seed_file(files, path: str, data: bytes = b"x") -> None:
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
files.downloads[path] = data
|
||||
files.metadata[path] = SimpleNamespace(is_directory=False,
|
||||
file_size=len(data))
|
||||
files.directories.setdefault(parent, [])
|
||||
files.directories[parent].append(
|
||||
SimpleNamespace(path=path, is_directory=False, file_size=len(data)))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rmdir_empty_succeeds(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/empty")
|
||||
|
||||
await rmdir(accessor, _path("/dbx/empty"), index)
|
||||
|
||||
assert files.delete_directory_calls == [f"{remote_root}/empty"]
|
||||
assert f"{remote_root}/empty" not in files.directory_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rmdir_non_empty_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/full")
|
||||
_seed_file(files, f"{remote_root}/full/a.txt")
|
||||
|
||||
with pytest.raises(OSError):
|
||||
await rmdir(accessor, _path("/dbx/full"), index)
|
||||
assert files.delete_directory_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rmdir_file_target_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/a.txt")
|
||||
|
||||
with pytest.raises(NotADirectoryError):
|
||||
await rmdir(accessor, _path("/dbx/a.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rmdir_missing_fails(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await rmdir(accessor, _path("/dbx/missing"), index)
|
||||
@@ -0,0 +1,311 @@
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.cache.index import RAMIndexCacheStore
|
||||
from mirage.core.databricks_volume.readdir import readdir
|
||||
from mirage.core.databricks_volume.stat import (_name_from_backend_path,
|
||||
modified_to_iso, stat)
|
||||
from mirage.types import FileType, PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
from .conftest import (ToThreadRecorder, directory_entry, file_entry,
|
||||
file_metadata)
|
||||
|
||||
|
||||
def test_modified_none_and_empty_string_return_none():
|
||||
assert modified_to_iso(None) is None
|
||||
assert modified_to_iso("") is None
|
||||
|
||||
|
||||
def test_modified_parses_http_date_to_iso_utc():
|
||||
assert modified_to_iso(
|
||||
"Tue, 14 Nov 2023 22:13:20 GMT") == "2023-11-14T22:13:20+00:00"
|
||||
|
||||
|
||||
def test_modified_returns_unparseable_string_verbatim():
|
||||
assert modified_to_iso("not a date") == "not a date"
|
||||
|
||||
|
||||
def test_modified_coerces_naive_datetime_to_utc():
|
||||
naive = datetime(2023, 11, 14, 22, 13, 20)
|
||||
assert modified_to_iso(naive) == "2023-11-14T22:13:20+00:00"
|
||||
|
||||
|
||||
def test_modified_converts_aware_datetime_to_utc():
|
||||
aware = datetime(2023, 11, 14, 22, 13, 20, tzinfo=timezone.utc)
|
||||
assert modified_to_iso(aware) == "2023-11-14T22:13:20+00:00"
|
||||
|
||||
|
||||
def test_modified_treats_large_int_as_epoch_milliseconds():
|
||||
assert modified_to_iso(1_700_000_000_000) == "2023-11-14T22:13:20+00:00"
|
||||
|
||||
|
||||
def test_name_from_backend_path_file():
|
||||
assert _name_from_backend_path(
|
||||
"/Volumes/main/default/agent_files/root/latest.md") == "latest.md"
|
||||
|
||||
|
||||
def test_name_from_backend_path_directory_with_trailing_slash():
|
||||
assert _name_from_backend_path(
|
||||
"/Volumes/main/default/agent_files/root/reports/") == "reports"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_file(accessor, files, remote_root):
|
||||
files.metadata[f"{remote_root}/reports/latest.md"] = file_metadata(
|
||||
size=6,
|
||||
modified="Tue, 14 Nov 2023 22:13:20 GMT",
|
||||
)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
result = await stat(accessor, path)
|
||||
assert result.name == "latest.md"
|
||||
assert result.size == 6
|
||||
assert result.modified == "2023-11-14T22:13:20+00:00"
|
||||
assert result.type != FileType.DIRECTORY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_file_from_index_skips_sdk(accessor, files, index,
|
||||
remote_root):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md",
|
||||
size=6,
|
||||
modified=1_700_000_000_000),
|
||||
]
|
||||
await readdir(
|
||||
accessor,
|
||||
PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume")), index)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
result = await stat(accessor, path, index)
|
||||
assert result.name == "latest.md"
|
||||
assert result.size == 6
|
||||
assert result.modified == "2023-11-14T22:13:20+00:00"
|
||||
assert result.type != FileType.DIRECTORY
|
||||
assert files.get_metadata_calls == []
|
||||
assert files.get_directory_metadata_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_directory_from_index_skips_sdk(accessor, files, index,
|
||||
remote_root):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
directory_entry(f"{remote_root}/reports/archive"),
|
||||
]
|
||||
await readdir(
|
||||
accessor,
|
||||
PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume")), index)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/archive",
|
||||
mount_key("/volume/reports/archive", "/volume"))
|
||||
result = await stat(accessor, path, index)
|
||||
assert result.name == "archive"
|
||||
assert result.type == FileType.DIRECTORY
|
||||
assert files.get_metadata_calls == []
|
||||
assert files.get_directory_metadata_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_index_negative_cache_raises_without_sdk(
|
||||
accessor,
|
||||
files,
|
||||
index,
|
||||
remote_root,
|
||||
):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md", size=6),
|
||||
]
|
||||
await readdir(
|
||||
accessor,
|
||||
PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume")), index)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/missing.md",
|
||||
mount_key("/volume/reports/missing.md", "/volume"))
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await stat(accessor, path, index)
|
||||
assert files.get_metadata_calls == []
|
||||
assert files.get_directory_metadata_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_index_fast_path_matches_sdk(accessor, files, index,
|
||||
remote_root):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
file_entry(f"{remote_root}/reports/latest.md",
|
||||
size=6,
|
||||
modified=1_700_000_000_000),
|
||||
]
|
||||
files.metadata[f"{remote_root}/reports/latest.md"] = file_metadata(
|
||||
size=6,
|
||||
modified="Tue, 14 Nov 2023 22:13:20 GMT",
|
||||
)
|
||||
await readdir(
|
||||
accessor,
|
||||
PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume")), index)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
fast = await stat(accessor, path, index)
|
||||
slow = await stat(accessor, path, RAMIndexCacheStore(ttl=600))
|
||||
assert fast == slow
|
||||
assert files.get_metadata_calls == [f"{remote_root}/reports/latest.md"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_directory_index_fast_path_matches_sdk(
|
||||
accessor,
|
||||
files,
|
||||
index,
|
||||
remote_root,
|
||||
):
|
||||
files.directories[f"{remote_root}/reports"] = [
|
||||
directory_entry(f"{remote_root}/reports/archive"),
|
||||
]
|
||||
files.directory_metadata.add(f"{remote_root}/reports/archive")
|
||||
await readdir(
|
||||
accessor,
|
||||
PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume")), index)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/archive",
|
||||
mount_key("/volume/reports/archive", "/volume"))
|
||||
fast = await stat(accessor, path, index)
|
||||
slow = await stat(accessor, path, RAMIndexCacheStore(ttl=600))
|
||||
assert fast == slow
|
||||
assert files.get_directory_metadata_calls == [
|
||||
f"{remote_root}/reports/archive"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_root_does_not_call_sdk(accessor, files):
|
||||
path = PathSpec.from_str_path("/volume", mount_key("/volume", "/volume"))
|
||||
result = await stat(accessor, path)
|
||||
assert result.name == "/"
|
||||
assert result.type == FileType.DIRECTORY
|
||||
assert files.get_metadata_calls == []
|
||||
assert files.get_directory_metadata_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_directory_uses_directory_metadata_fallback(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
files.directory_metadata.add(f"{remote_root}/reports")
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
result = await stat(accessor, path)
|
||||
assert result.name == "reports"
|
||||
assert result.size is None
|
||||
assert result.type == FileType.DIRECTORY
|
||||
assert files.get_metadata_calls == [f"{remote_root}/reports"]
|
||||
assert files.get_directory_metadata_calls == [f"{remote_root}/reports"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_missing_path_raises(accessor):
|
||||
path = PathSpec.from_str_path("/volume/missing",
|
||||
mount_key("/volume/missing", "/volume"))
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await stat(accessor, path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_missing_path_checks_directory_metadata(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
path = PathSpec.from_str_path("/volume/missing",
|
||||
mount_key("/volume/missing", "/volume"))
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await stat(accessor, path)
|
||||
assert files.get_metadata_calls == [f"{remote_root}/missing"]
|
||||
assert files.get_directory_metadata_calls == [f"{remote_root}/missing"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_rejects_path_escape(accessor):
|
||||
path = PathSpec.from_str_path("/volume/../outside",
|
||||
mount_key("/volume/../outside", "/volume"))
|
||||
with pytest.raises(ValueError, match="escapes Databricks volume root"):
|
||||
await stat(accessor, path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_metadata_error_propagates(accessor, files, remote_root):
|
||||
remote_path = f"{remote_root}/reports"
|
||||
files.metadata_errors[remote_path] = RuntimeError("metadata failed")
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
with pytest.raises(RuntimeError, match="metadata failed"):
|
||||
await stat(accessor, path)
|
||||
assert files.get_metadata_calls == [remote_path]
|
||||
assert files.get_directory_metadata_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_directory_metadata_error_propagates(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
remote_path = f"{remote_root}/reports"
|
||||
files.directory_metadata_errors[remote_path] = RuntimeError(
|
||||
"directory metadata failed")
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
with pytest.raises(RuntimeError, match="directory metadata failed"):
|
||||
await stat(accessor, path)
|
||||
assert files.get_metadata_calls == [remote_path]
|
||||
assert files.get_directory_metadata_calls == [remote_path]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_runs_blocking_metadata_off_event_loop(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
monkeypatch,
|
||||
):
|
||||
to_thread = ToThreadRecorder()
|
||||
monkeypatch.setattr(asyncio, "to_thread", to_thread)
|
||||
files.metadata[f"{remote_root}/reports/latest.md"] = file_metadata(size=6)
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
result = await stat(accessor, path)
|
||||
|
||||
assert result.name == "latest.md"
|
||||
assert len(to_thread.calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stat_directory_fallback_runs_off_event_loop(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
monkeypatch,
|
||||
):
|
||||
to_thread = ToThreadRecorder()
|
||||
monkeypatch.setattr(asyncio, "to_thread", to_thread)
|
||||
files.directory_metadata.add(f"{remote_root}/reports")
|
||||
path = PathSpec.from_str_path("/volume/reports",
|
||||
mount_key("/volume/reports", "/volume"))
|
||||
|
||||
result = await stat(accessor, path)
|
||||
|
||||
assert result.type == FileType.DIRECTORY
|
||||
assert len(to_thread.calls) == 2
|
||||
@@ -0,0 +1,114 @@
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.stream import range_read, read_stream
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
class TrackingContents:
|
||||
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
self.offset = 0
|
||||
self.read_sizes: list[int] = []
|
||||
self.closed = False
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
self.read_sizes.append(size)
|
||||
if size < 0:
|
||||
size = len(self.data) - self.offset
|
||||
chunk = self.data[self.offset:self.offset + size]
|
||||
self.offset += len(chunk)
|
||||
return chunk
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class TrackingDownload:
|
||||
|
||||
def __init__(self, contents: TrackingContents) -> None:
|
||||
self.contents = contents
|
||||
|
||||
|
||||
class TrackingFiles:
|
||||
|
||||
def __init__(self, contents: TrackingContents) -> None:
|
||||
self.contents = contents
|
||||
self.download_calls: list[str] = []
|
||||
|
||||
def download(self, path: str) -> TrackingDownload:
|
||||
self.download_calls.append(path)
|
||||
return TrackingDownload(self.contents)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_stream_chunks_file(accessor, files, remote_root):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
chunks = [
|
||||
chunk async for chunk in read_stream(accessor, path, chunk_size=2)
|
||||
]
|
||||
assert chunks == [b"ab", b"cd", b"ef"]
|
||||
# Streaming should use one download body, not one Range GET per chunk.
|
||||
assert files.download_calls == [f"{remote_root}/reports/latest.md"]
|
||||
assert accessor.client.api_client.do_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_range_read_uses_end_exclusive(accessor, files, remote_root):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
result = await range_read(accessor, path, 1, 4)
|
||||
assert result == b"bcd"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_range_read_uses_single_databricks_range_request(
|
||||
accessor,
|
||||
files,
|
||||
remote_root,
|
||||
):
|
||||
files.downloads[f"{remote_root}/reports/latest.md"] = b"abcdef"
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
|
||||
result = await range_read(accessor, path, 1, 4)
|
||||
|
||||
assert result == b"bcd"
|
||||
assert files.download_calls == []
|
||||
assert len(accessor.client.api_client.do_calls) == 1
|
||||
assert accessor.client.api_client.do_calls[0]["headers"]["Range"] == (
|
||||
"bytes=1-3")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_stream_reads_single_download_body_in_chunks(
|
||||
accessor,
|
||||
remote_root,
|
||||
):
|
||||
contents = TrackingContents(b"abcdef")
|
||||
tracking_files = TrackingFiles(contents)
|
||||
accessor.client.files = tracking_files
|
||||
path = PathSpec.from_str_path(
|
||||
"/volume/reports/latest.md",
|
||||
mount_key("/volume/reports/latest.md", "/volume"))
|
||||
stream = read_stream(accessor, path, chunk_size=2)
|
||||
|
||||
first = await anext(stream)
|
||||
second = await anext(stream)
|
||||
|
||||
assert first == b"ab"
|
||||
assert second == b"cd"
|
||||
assert tracking_files.download_calls == [
|
||||
f"{remote_root}/reports/latest.md"
|
||||
]
|
||||
assert contents.read_sizes == [2, 2]
|
||||
assert not contents.closed
|
||||
await stream.aclose()
|
||||
assert contents.closed
|
||||
@@ -0,0 +1,158 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import pytest
|
||||
|
||||
from mirage.core.databricks_volume.create import create
|
||||
from mirage.core.databricks_volume.unlink import unlink
|
||||
from mirage.core.databricks_volume.write import write_bytes
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_key
|
||||
|
||||
|
||||
def _path(path: str) -> PathSpec:
|
||||
return PathSpec.from_str_path(path, mount_key(path, "/dbx"))
|
||||
|
||||
|
||||
def _seed_directory(files, path: str) -> None:
|
||||
files.directory_metadata.add(path)
|
||||
files.metadata[path] = type("Metadata", (), {"is_directory": True})()
|
||||
files.directories.setdefault(path, [])
|
||||
|
||||
|
||||
def _seed_file(files, path: str, data: bytes) -> None:
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
files.downloads[path] = data
|
||||
files.metadata[path] = type(
|
||||
"Metadata",
|
||||
(),
|
||||
{
|
||||
"content_length": len(data),
|
||||
},
|
||||
)()
|
||||
files.directories.setdefault(parent, [])
|
||||
files.directories[parent].append(
|
||||
type(
|
||||
"Entry",
|
||||
(),
|
||||
{
|
||||
"path": path,
|
||||
"is_directory": False,
|
||||
"file_size": len(data),
|
||||
},
|
||||
)())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_new_file(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
await write_bytes(accessor, _path("/dbx/new.txt"), b"hello", index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/new.txt"] == b"hello"
|
||||
assert files.upload_calls == [(f"{remote_root}/new.txt", b"hello", True)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_overwrites_existing_file(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/new.txt", b"old")
|
||||
|
||||
await write_bytes(accessor, _path("/dbx/new.txt"), b"new", index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/new.txt"] == b"new"
|
||||
assert files.metadata[f"{remote_root}/new.txt"].content_length == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_fails_when_parent_is_missing(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await write_bytes(accessor, _path("/dbx/missing/new.txt"), b"x", index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_fails_when_parent_is_file(accessor, files, remote_root,
|
||||
index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/parent.txt", b"file")
|
||||
|
||||
with pytest.raises(NotADirectoryError):
|
||||
await write_bytes(accessor, _path("/dbx/parent.txt/new.txt"), b"x",
|
||||
index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_empty_file(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
await create(accessor, _path("/dbx/empty.txt"), index)
|
||||
|
||||
assert files.downloads[f"{remote_root}/empty.txt"] == b""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_fails_when_parent_is_missing(accessor, files,
|
||||
remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await create(accessor, _path("/dbx/missing/empty.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unlink_file(accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_file(files, f"{remote_root}/delete.txt", b"bye")
|
||||
|
||||
await unlink(accessor, _path("/dbx/delete.txt"), index)
|
||||
|
||||
assert f"{remote_root}/delete.txt" not in files.downloads
|
||||
assert files.delete_calls == [f"{remote_root}/delete.txt"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unlink_missing_file_raises_file_not_found(
|
||||
accessor, files, remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await unlink(accessor, _path("/dbx/missing.txt"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unlink_directory_raises_is_a_directory(accessor, files,
|
||||
remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
_seed_directory(files, f"{remote_root}/dir")
|
||||
|
||||
with pytest.raises(IsADirectoryError):
|
||||
await unlink(accessor, _path("/dbx/dir"), index)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_mutation_paths_cannot_escape_root(accessor, files,
|
||||
remote_root, index):
|
||||
_seed_directory(files, remote_root)
|
||||
escaping = _path("/dbx/../escape.txt")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await write_bytes(accessor, escaping, b"x", index)
|
||||
with pytest.raises(ValueError):
|
||||
await create(accessor, escaping, index)
|
||||
with pytest.raises(ValueError):
|
||||
await unlink(accessor, escaping, index)
|
||||
Reference in New Issue
Block a user