chore: import upstream snapshot with attribution
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
Integ / changes (push) Has been skipped
Pre-commit / pre-commit (push) Failing after 1s
CLI exit codes / changes (push) Has been skipped
Test (Install) / changes (push) Has been skipped
Test (Python) / changes (push) Has been skipped
Test (TypeScript) / changes (push) Has been skipped
CLI exit codes / cli-gate (push) Has been cancelled
Test (Install) / test-install-gate (push) Has been cancelled
Integ / integ-gate (push) Has been cancelled
Test (Python) / test-python-gate (push) Has been cancelled
Test (TypeScript) / test-typescript-gate (push) Has been cancelled
Test (Install) / python-minimal (3.12) (push) Has been cancelled
Test (Install) / python-minimal (3.11) (push) Has been cancelled
Test (Install) / python-extra (agno, mirage.agents.agno) (push) Has been cancelled
Test (Install) / python-extra (chroma, mirage.resource.chroma) (push) Has been cancelled
Test (Install) / python-extra (pdf, mirage.core.filetype.pdf) (push) Has been cancelled
Integ / integ (push) Has been cancelled
Integ / integ-database (push) Has been cancelled
Integ / integ-database-ts (push) Has been cancelled
Integ / integ-data (push) Has been cancelled
Integ / integ-ssh (push) Has been cancelled
Integ / integ-ssh-ts (push) Has been cancelled
Test (Python) / audit (push) Has been cancelled
Test (TypeScript) / test (push) Has been cancelled
Test (TypeScript) / python-fs-shim (push) Has been cancelled
CLI exit codes / Python CLI (push) Has been cancelled
CLI exit codes / TypeScript CLI (push) Has been cancelled
CLI exit codes / Cross-language snapshot interop (push) Has been cancelled
Test (Python) / test (push) Has been cancelled
Test (Python) / import-isolation (deepagents, openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Python) / import-isolation (deepagents, pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Integ / integ-ts (push) Has been cancelled
Integ / integ-fuse (push) Has been cancelled
Test (Install) / python-extra (databricks, mirage.resource.databricks_volume) (push) Has been cancelled
Test (Install) / python-extra (deepagents, mirage.agents.langchain) (push) Has been cancelled
Test (Install) / python-extra (email, mirage.resource.email) (push) Has been cancelled
Test (Install) / python-extra (fuse, mirage.fuse.mount) (push) Has been cancelled
Test (Install) / python-extra (hdf5, mirage.core.filetype.hdf5) (push) Has been cancelled
Test (Install) / python-extra (hf, mirage.resource.hf_buckets) (push) Has been cancelled
Test (Install) / python-extra (lancedb, mirage.resource.lancedb) (push) Has been cancelled
Test (Install) / python-extra (langfuse, mirage.resource.langfuse) (push) Has been cancelled
Test (Install) / python-extra (mongodb, mirage.resource.mongodb) (push) Has been cancelled
Test (Install) / python-extra (nextcloud, mirage.resource.nextcloud) (push) Has been cancelled
Test (Install) / python-extra (openai, mirage.agents.openai_agents) (push) Has been cancelled
Test (Install) / python-extra (openhands, mirage.agents.openhands, 3.12) (push) Has been cancelled
Test (Install) / python-extra (parquet, mirage.core.filetype.parquet) (push) Has been cancelled
Test (Install) / python-extra (postgres, mirage.resource.postgres) (push) Has been cancelled
Test (Install) / python-extra (pydantic-ai, mirage.agents.pydantic_ai) (push) Has been cancelled
Test (Install) / python-extra (qdrant, mirage.resource.qdrant) (push) Has been cancelled
Test (Install) / python-extra (redis, mirage.resource.redis) (push) Has been cancelled
Test (Install) / python-extra (s3, mirage.resource.s3) (push) Has been cancelled
Test (Install) / python-extra (ssh, mirage.resource.ssh) (push) Has been cancelled
Test (Install) / ts-minimal (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.core.disk.copy import copy
|
||||
from mirage.core.disk.create import create
|
||||
from mirage.core.disk.du import du, du_all
|
||||
from mirage.core.disk.exists import exists
|
||||
from mirage.core.disk.find import find
|
||||
from mirage.core.disk.mkdir import mkdir
|
||||
from mirage.core.disk.read import read_bytes
|
||||
from mirage.core.disk.readdir import readdir
|
||||
from mirage.core.disk.rename import rename
|
||||
from mirage.core.disk.rm import rm_r
|
||||
from mirage.core.disk.rmdir import rmdir
|
||||
from mirage.core.disk.stat import stat
|
||||
from mirage.core.disk.stream import read_stream
|
||||
from mirage.core.disk.truncate import truncate
|
||||
from mirage.core.disk.unlink import unlink
|
||||
from mirage.core.disk.write import write_bytes
|
||||
|
||||
__all__ = [
|
||||
"copy",
|
||||
"create",
|
||||
"du",
|
||||
"du_all",
|
||||
"exists",
|
||||
"find",
|
||||
"mkdir",
|
||||
"read_bytes",
|
||||
"read_stream",
|
||||
"readdir",
|
||||
"rename",
|
||||
"rm_r",
|
||||
"rmdir",
|
||||
"stat",
|
||||
"truncate",
|
||||
"unlink",
|
||||
"write_bytes",
|
||||
]
|
||||
@@ -0,0 +1,48 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_write
|
||||
from mirage.observe.context import record
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def append_bytes(accessor: DiskAccessor, path: PathSpec,
|
||||
data: bytes) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
root = accessor.root
|
||||
start_ms = int(time.monotonic() * 1000)
|
||||
p = _resolve(root, path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.open(p, "ab") as f:
|
||||
await f.write(data)
|
||||
record("append", path, "disk", len(data), start_ms)
|
||||
await invalidate_after_write(path)
|
||||
@@ -0,0 +1,16 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
SCOPE_WARN = 5000
|
||||
SCOPE_ERROR = 50000
|
||||
@@ -0,0 +1,51 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_write
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def copy(accessor: DiskAccessor, src: PathSpec, dst: PathSpec) -> None:
|
||||
if isinstance(src, str):
|
||||
src = PathSpec(virtual=src,
|
||||
directory=src,
|
||||
resource_path=src.strip("/"))
|
||||
if isinstance(src, PathSpec):
|
||||
src = src.mount_path
|
||||
if isinstance(dst, str):
|
||||
dst = PathSpec(virtual=dst,
|
||||
directory=dst,
|
||||
resource_path=dst.strip("/"))
|
||||
if isinstance(dst, PathSpec):
|
||||
dst = dst.mount_path
|
||||
root = accessor.root
|
||||
s = _resolve(root, src)
|
||||
d = _resolve(root, dst)
|
||||
await aiofiles.os.makedirs(d.parent, exist_ok=True)
|
||||
await asyncio.to_thread(shutil.copy2, s, d)
|
||||
await invalidate_after_write(dst)
|
||||
@@ -0,0 +1,43 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
import aiofiles.os
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_write
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def create(accessor: DiskAccessor, path: PathSpec) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
await aiofiles.os.makedirs(p.parent, exist_ok=True)
|
||||
async with aiofiles.open(p, "wb") as f:
|
||||
await f.write(b"")
|
||||
await invalidate_after_write(path)
|
||||
@@ -0,0 +1,86 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
def _du_sync(root: Path, path: str) -> int:
|
||||
p = _resolve(root, path)
|
||||
if p.is_file():
|
||||
return p.stat().st_size
|
||||
total = 0
|
||||
for dirpath, _dirnames, filenames in os.walk(p):
|
||||
for f in filenames:
|
||||
try:
|
||||
total += os.path.getsize(os.path.join(dirpath, f))
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def _du_all_sync(root: Path, path: str) -> tuple[list[tuple[str, int]], int]:
|
||||
p = _resolve(root, path)
|
||||
if p.is_file():
|
||||
size = p.stat().st_size
|
||||
return [(("/" + path.strip("/")), size)], size
|
||||
entries: list[tuple[str, int]] = []
|
||||
total = 0
|
||||
for dirpath, _dirnames, filenames in os.walk(p):
|
||||
for f in filenames:
|
||||
full = os.path.join(dirpath, f)
|
||||
try:
|
||||
size = os.path.getsize(full)
|
||||
except OSError:
|
||||
continue
|
||||
rel = os.path.relpath(full, root)
|
||||
entry_path = "/" + rel
|
||||
entries.append((entry_path, size))
|
||||
total += size
|
||||
entries.sort()
|
||||
return entries, total
|
||||
|
||||
|
||||
async def du(accessor: DiskAccessor, path: PathSpec) -> int:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
return await asyncio.to_thread(_du_sync, accessor.root, path)
|
||||
|
||||
|
||||
async def du_all(
|
||||
accessor: DiskAccessor,
|
||||
path: PathSpec,
|
||||
) -> tuple[list[tuple[str, int]], int]:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
return await asyncio.to_thread(_du_all_sync, accessor.root, path)
|
||||
@@ -0,0 +1,38 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from aiofiles.os import path as aio_path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def exists(accessor: DiskAccessor, path: PathSpec) -> bool:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
return await aio_path.exists(p)
|
||||
@@ -0,0 +1,197 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.commands.builtin.find_eval import (FindEntry, PredNode, build_tree,
|
||||
emit_start_path, keep,
|
||||
start_basename)
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
def _find_sync(
|
||||
root: Path,
|
||||
path: str,
|
||||
name: str | None = None,
|
||||
type: str | None = None,
|
||||
min_size: int | None = None,
|
||||
max_size: int | None = None,
|
||||
maxdepth: int | None = None,
|
||||
name_exclude: str | None = None,
|
||||
or_names: list[str] | None = None,
|
||||
mtime_min: float | None = None,
|
||||
mtime_max: float | None = None,
|
||||
iname: str | None = None,
|
||||
path_pattern: str | None = None,
|
||||
mindepth: int | None = None,
|
||||
empty: bool = False,
|
||||
tree: PredNode | None = None,
|
||||
start_name: str = "",
|
||||
) -> list[str]:
|
||||
p = _resolve(root, path)
|
||||
base = "/" + path.strip("/")
|
||||
base_depth = 0 if base == "/" else base.count("/")
|
||||
results: list[str] = []
|
||||
tree = tree if tree is not None else build_tree(name=name,
|
||||
iname=iname,
|
||||
path_pattern=path_pattern,
|
||||
type=type,
|
||||
name_exclude=name_exclude,
|
||||
or_names=or_names,
|
||||
empty=empty)
|
||||
|
||||
if p.is_dir():
|
||||
root_empty = (not any(p.iterdir())) if empty else None
|
||||
emit_start_path(results,
|
||||
base,
|
||||
start_name,
|
||||
kind="d",
|
||||
is_empty=root_empty,
|
||||
exists=True,
|
||||
tree=tree,
|
||||
maxdepth=maxdepth,
|
||||
mindepth=mindepth,
|
||||
min_size=min_size,
|
||||
max_size=max_size)
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(p):
|
||||
dp = Path(dirpath)
|
||||
rel = dp.relative_to(root)
|
||||
current = "/" + str(rel) if str(rel) != "." else "/"
|
||||
|
||||
current_depth = current.count("/") - base_depth
|
||||
|
||||
if maxdepth is not None and current_depth > maxdepth:
|
||||
dirnames.clear()
|
||||
continue
|
||||
|
||||
entries: list[tuple[str, str]] = []
|
||||
if type != "f" and type != "file":
|
||||
for d in dirnames:
|
||||
entry_path = current.rstrip("/") + "/" + d
|
||||
entries.append((entry_path, "d"))
|
||||
if type != "d" and type != "directory":
|
||||
for f in filenames:
|
||||
entry_path = current.rstrip("/") + "/" + f
|
||||
entries.append((entry_path, "f"))
|
||||
|
||||
for entry_path, kind in entries:
|
||||
entry_name = entry_path.rsplit("/", 1)[-1]
|
||||
depth = entry_path.count("/") - base_depth
|
||||
if maxdepth is not None and depth > maxdepth:
|
||||
continue
|
||||
|
||||
full = root / entry_path.lstrip("/")
|
||||
is_empty: bool | None = None
|
||||
if empty:
|
||||
try:
|
||||
is_empty = (full.stat().st_size == 0) if kind == "f" else (
|
||||
not any(full.iterdir()))
|
||||
except OSError:
|
||||
is_empty = None
|
||||
entry = FindEntry(key=entry_path,
|
||||
name=entry_name,
|
||||
kind=kind,
|
||||
depth=depth,
|
||||
is_empty=is_empty)
|
||||
if not keep(entry, tree, mindepth):
|
||||
continue
|
||||
|
||||
# Directories count as size 0 for -size (deliberate GNU
|
||||
# divergence).
|
||||
if min_size is not None or max_size is not None:
|
||||
if kind == "f":
|
||||
try:
|
||||
size = full.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
else:
|
||||
size = 0
|
||||
if min_size is not None and size < min_size:
|
||||
continue
|
||||
if max_size is not None and size > max_size:
|
||||
continue
|
||||
|
||||
if mtime_min is not None or mtime_max is not None:
|
||||
try:
|
||||
st = full.stat()
|
||||
mtime = datetime.fromtimestamp(
|
||||
st.st_mtime, tz=timezone.utc).timestamp()
|
||||
except OSError:
|
||||
continue
|
||||
if mtime_min is not None and mtime < mtime_min:
|
||||
continue
|
||||
if mtime_max is not None and mtime > mtime_max:
|
||||
continue
|
||||
|
||||
results.append(entry_path)
|
||||
|
||||
return sorted(results)
|
||||
|
||||
|
||||
async def find(
|
||||
accessor: DiskAccessor,
|
||||
path: PathSpec,
|
||||
name: str | None = None,
|
||||
type: str | None = None,
|
||||
min_size: int | None = None,
|
||||
max_size: int | None = None,
|
||||
maxdepth: int | None = None,
|
||||
name_exclude: str | None = None,
|
||||
or_names: list[str] | None = None,
|
||||
mtime_min: float | None = None,
|
||||
mtime_max: float | None = None,
|
||||
iname: str | None = None,
|
||||
path_pattern: str | None = None,
|
||||
mindepth: int | None = None,
|
||||
empty: bool = False,
|
||||
tree: PredNode | None = None,
|
||||
) -> list[str]:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
start_name = start_basename(path)
|
||||
path = path.mount_path
|
||||
return await asyncio.to_thread(
|
||||
_find_sync,
|
||||
accessor.root,
|
||||
path,
|
||||
name,
|
||||
type,
|
||||
min_size,
|
||||
max_size,
|
||||
maxdepth,
|
||||
name_exclude,
|
||||
or_names,
|
||||
mtime_min,
|
||||
mtime_max,
|
||||
iname,
|
||||
path_pattern,
|
||||
mindepth,
|
||||
empty,
|
||||
tree,
|
||||
start_name,
|
||||
)
|
||||
@@ -0,0 +1,29 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.index import IndexCacheStore
|
||||
from mirage.core.disk.constants import SCOPE_ERROR
|
||||
from mirage.core.disk.readdir import readdir
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.glob_walk import resolve_glob_with
|
||||
|
||||
|
||||
async def resolve_glob(
|
||||
accessor: DiskAccessor,
|
||||
paths: list[PathSpec],
|
||||
index: IndexCacheStore,
|
||||
) -> list[PathSpec]:
|
||||
return await resolve_glob_with(readdir, accessor, paths, index,
|
||||
SCOPE_ERROR)
|
||||
@@ -0,0 +1,48 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_write
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def mkdir(accessor: DiskAccessor,
|
||||
path: PathSpec,
|
||||
parents: bool = False) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
if parents:
|
||||
await aiofiles.os.makedirs(p, exist_ok=True)
|
||||
else:
|
||||
try:
|
||||
await aiofiles.os.mkdir(p)
|
||||
except FileExistsError:
|
||||
pass
|
||||
await invalidate_after_write(path)
|
||||
@@ -0,0 +1,52 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.index import IndexCacheStore
|
||||
from mirage.observe.context import record
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def read_bytes(accessor: DiskAccessor,
|
||||
path: PathSpec,
|
||||
index: IndexCacheStore = None) -> bytes:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
virtual = path.virtual
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
root = accessor.root
|
||||
start_ms = int(time.monotonic() * 1000)
|
||||
p = _resolve(root, path)
|
||||
try:
|
||||
async with aiofiles.open(p, "rb") as f:
|
||||
data = await f.read()
|
||||
except FileNotFoundError as exc:
|
||||
raise FileNotFoundError(virtual) from exc
|
||||
record("read", path, "disk", len(data), start_ms)
|
||||
return data
|
||||
@@ -0,0 +1,66 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
from aiofiles.os import path as aio_path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.index import IndexCacheStore, IndexEntry
|
||||
from mirage.types import PathSpec
|
||||
from mirage.utils.key_prefix import mount_prefix_of
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def readdir(accessor: DiskAccessor, path: PathSpec,
|
||||
index: IndexCacheStore) -> list[str]:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
prefix = mount_prefix_of(path.virtual, path.resource_path)
|
||||
path = path.directory if path.pattern else path.virtual
|
||||
if prefix and path.startswith(prefix):
|
||||
rest = path[len(prefix):]
|
||||
if prefix.endswith("/") or rest == "" or rest.startswith("/"):
|
||||
path = rest or "/"
|
||||
root = accessor.root
|
||||
# Canonical key: no trailing slash (except root), or the same dir
|
||||
# indexes under two keys and cache hits return doubled-slash entries.
|
||||
virtual_key = prefix + path if prefix else path
|
||||
virtual_key = virtual_key.rstrip("/") or "/"
|
||||
listing = await index.list_dir(virtual_key)
|
||||
if listing.entries is not None:
|
||||
return listing.entries
|
||||
p = _resolve(root, path)
|
||||
if not await aio_path.isdir(p):
|
||||
raise NotADirectoryError(str(p))
|
||||
base = "/" + path.strip("/")
|
||||
raw = await aiofiles.os.listdir(p)
|
||||
entries = sorted(base.rstrip("/") + "/" + name for name in raw)
|
||||
virtual_entries = sorted((prefix + e if prefix else e) for e in entries)
|
||||
index_entries = [(e.rsplit("/", 1)[-1],
|
||||
IndexEntry(id=e,
|
||||
name=e.rsplit("/", 1)[-1],
|
||||
resource_type="file")) for e in entries]
|
||||
await index.set_dir(virtual_key, index_entries)
|
||||
return virtual_entries
|
||||
@@ -0,0 +1,48 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import (invalidate_after_unlink,
|
||||
invalidate_after_write)
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def rename(accessor: DiskAccessor, src: PathSpec, dst: PathSpec) -> None:
|
||||
if isinstance(src, str):
|
||||
src = PathSpec(virtual=src,
|
||||
directory=src,
|
||||
resource_path=src.strip("/"))
|
||||
if isinstance(src, PathSpec):
|
||||
src = src.mount_path
|
||||
if isinstance(dst, str):
|
||||
dst = PathSpec(virtual=dst,
|
||||
directory=dst,
|
||||
resource_path=dst.strip("/"))
|
||||
if isinstance(dst, PathSpec):
|
||||
dst = dst.mount_path
|
||||
root = accessor.root
|
||||
await invalidate_after_unlink(src)
|
||||
await invalidate_after_write(dst)
|
||||
await aiofiles.os.rename(_resolve(root, src), _resolve(root, dst))
|
||||
@@ -0,0 +1,46 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
from aiofiles.os import path as aio_path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_unlink
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def rm_r(accessor: DiskAccessor, path: PathSpec) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
if await aio_path.isdir(p):
|
||||
await asyncio.to_thread(shutil.rmtree, p)
|
||||
elif await aio_path.exists(p):
|
||||
await aiofiles.os.remove(p)
|
||||
await invalidate_after_unlink(path)
|
||||
@@ -0,0 +1,40 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_unlink
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def rmdir(accessor: DiskAccessor, path: PathSpec) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
await aiofiles.os.rmdir(p)
|
||||
await invalidate_after_unlink(path)
|
||||
@@ -0,0 +1,59 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
from aiofiles.os import path as aio_path
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.index import IndexCacheStore
|
||||
from mirage.core.timeutil import epoch_to_iso
|
||||
from mirage.types import FileStat, FileType, PathSpec
|
||||
from mirage.utils.filetype import guess_type
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def stat(accessor: DiskAccessor,
|
||||
path: PathSpec,
|
||||
index: IndexCacheStore = None) -> FileStat:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
virtual = path.virtual
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
root = accessor.root
|
||||
p = _resolve(root, path)
|
||||
if not await aio_path.exists(p):
|
||||
raise FileNotFoundError(virtual)
|
||||
st = await aiofiles.os.stat(p)
|
||||
modified = epoch_to_iso(st.st_mtime)
|
||||
if await aio_path.isdir(p):
|
||||
return FileStat(name=p.name,
|
||||
size=None,
|
||||
modified=modified,
|
||||
type=FileType.DIRECTORY)
|
||||
return FileStat(name=p.name,
|
||||
size=st.st_size,
|
||||
modified=modified,
|
||||
fingerprint=modified,
|
||||
type=guess_type(p.name))
|
||||
@@ -0,0 +1,57 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.index import IndexCacheStore
|
||||
from mirage.observe.context import record_stream
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def read_stream(accessor: DiskAccessor,
|
||||
path: PathSpec,
|
||||
index: IndexCacheStore = None,
|
||||
chunk_size: int = 8192) -> AsyncIterator[bytes]:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
virtual = path.virtual
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
root = accessor.root
|
||||
rec = record_stream("read", path, "disk")
|
||||
p = _resolve(root, path)
|
||||
try:
|
||||
async with aiofiles.open(p, "rb") as f:
|
||||
while True:
|
||||
chunk = await f.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
if rec is not None:
|
||||
rec.bytes += len(chunk)
|
||||
yield chunk
|
||||
except FileNotFoundError as exc:
|
||||
raise FileNotFoundError(virtual) from exc
|
||||
@@ -0,0 +1,48 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_write
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def truncate(accessor: DiskAccessor, path: PathSpec,
|
||||
length: int) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
try:
|
||||
async with aiofiles.open(p, "rb") as f:
|
||||
data = await f.read()
|
||||
except FileNotFoundError:
|
||||
data = b""
|
||||
result = data[:length].ljust(length, b"\0")
|
||||
async with aiofiles.open(p, "wb") as f:
|
||||
await f.write(result)
|
||||
await invalidate_after_write(path)
|
||||
@@ -0,0 +1,40 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles.os
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_unlink
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def unlink(accessor: DiskAccessor, path: PathSpec) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
p = _resolve(accessor.root, path)
|
||||
await aiofiles.os.remove(p)
|
||||
await invalidate_after_unlink(path)
|
||||
@@ -0,0 +1,48 @@
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiofiles
|
||||
|
||||
from mirage.accessor.disk import DiskAccessor
|
||||
from mirage.cache.context import invalidate_after_write
|
||||
from mirage.observe.context import record
|
||||
from mirage.types import PathSpec
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path:
|
||||
relative = path.lstrip("/")
|
||||
resolved = (root / relative).resolve()
|
||||
resolved.relative_to(root)
|
||||
return resolved
|
||||
|
||||
|
||||
async def write_bytes(accessor: DiskAccessor, path: PathSpec,
|
||||
data: bytes) -> None:
|
||||
if isinstance(path, str):
|
||||
path = PathSpec(virtual=path,
|
||||
directory=path,
|
||||
resource_path=path.strip("/"))
|
||||
if isinstance(path, PathSpec):
|
||||
path = path.mount_path
|
||||
root = accessor.root
|
||||
start_ms = int(time.monotonic() * 1000)
|
||||
p = _resolve(root, path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.open(p, "wb") as f:
|
||||
await f.write(data)
|
||||
record("write", path, "disk", len(data), start_ms)
|
||||
await invalidate_after_write(path)
|
||||
Reference in New Issue
Block a user