chore: import upstream snapshot with attribution

python/mirage/cache/file/__init__.py

@@ -0,0 +1,18 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from mirage.cache.file.entry import CacheEntry
+from mirage.cache.file.mixin import FileCacheMixin
+
+__all__ = ["CacheEntry", "FileCacheMixin"]

python/mirage/cache/file/config.py

@@ -0,0 +1,29 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from pydantic import BaseModel
+
+from mirage.types import CacheType
+
+
+class CacheConfig(BaseModel):
+    type: CacheType = CacheType.RAM
+    limit: str | int = "512MB"
+    max_drain_bytes: int | None = None
+
+
+class RedisCacheConfig(CacheConfig):
+    type: CacheType = CacheType.REDIS
+    url: str = "redis://localhost:6379/0"
+    key_prefix: str = "mirage:cache:"

python/mirage/cache/file/entry.py

@@ -0,0 +1,30 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import time
+
+from pydantic import BaseModel
+
+
+class CacheEntry(BaseModel):
+    size: int
+    cached_at: int
+    fingerprint: str | None = None
+    ttl: int | None = None
+
+    @property
+    def expired(self) -> bool:
+        if self.ttl is None:
+            return False
+        return (int(time.time()) - self.cached_at) >= self.ttl

python/mirage/cache/file/io.py

@@ -0,0 +1,139 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import logging
+from typing import Callable
+
+from mirage.cache.file.mixin import FileCacheMixin
+from mirage.io import CachableAsyncIterator, IOResult
+from mirage.observe.record import OpRecord
+
+logger = logging.getLogger(__name__)
+
+
+def read_fingerprint(records: list[OpRecord] | None, path: str) -> str | None:
+    """Latest backend fingerprint recorded for a read of ``path``.
+
+    Backends stamp read records with the content identifier they
+    returned (S3 ETag, OneDrive cTag, Postgres sha256). Threading it
+    into the cache entry lets ALWAYS-mode ``is_fresh`` compare like
+    with like; the MD5-of-content default only matches simple-PUT S3
+    objects.
+
+    Args:
+        records (list[OpRecord] | None): Op records emitted by the
+            command that produced the IOResult being applied.
+        path (str): Virtual path used as the cache key.
+    """
+    if records is None:
+        return None
+    for rec in reversed(records):
+        if rec.op == "read" and rec.path == path and rec.fingerprint:
+            return rec.fingerprint
+    return None
+
+
+async def _set_cached(
+    cache: FileCacheMixin,
+    path: str,
+    data: bytes,
+    records: list[OpRecord] | None,
+) -> None:
+    fingerprint = read_fingerprint(records, path)
+    if fingerprint is None and await cache.get(path) == data:
+        # Warm read: the bytes were served from this cache, so there is
+        # no backend read record. Re-setting would replace the backend
+        # fingerprint stamped on the cold read with the MD5 default and
+        # force ALWAYS mode to evict and refetch on every read.
+        return
+    await cache.set(path, data, fingerprint=fingerprint)
+
+
+async def apply_io(
+    cache: FileCacheMixin,
+    io: IOResult,
+    is_cacheable: Callable[[str], bool] | None = None,
+    records: list[OpRecord] | None = None,
+) -> None:
+    cache_set = set(io.cache)
+    max_bytes = getattr(cache, "max_drain_bytes", None)
+    for path in io.cache:
+        if is_cacheable is not None and not is_cacheable(path):
+            continue
+        data = io.reads.get(path)
+        if data is None:
+            data = io.writes.get(path)
+        if data is None:
+            continue
+        if isinstance(data, bytes):
+            await _set_cached(cache, path, data, records)
+        elif isinstance(data, CachableAsyncIterator):
+            if data.exhausted:
+                await _set_cached(cache, path, b"".join(data.buffered_chunks),
+                                  records)
+            else:
+                if (hasattr(cache, "_drain_tasks")
+                        and path not in cache._drain_tasks
+                        and not await cache.exists(path)):
+                    task = asyncio.create_task(
+                        _background_drain(cache, path, data, max_bytes,
+                                          records))
+                    cache._drain_tasks[path] = task
+                    task.add_done_callback(
+                        lambda t, p=path: cache._drain_tasks.pop(p, None))
+    for path in io.writes:
+        if path in cache_set:
+            continue
+        if is_cacheable is not None and not is_cacheable(path):
+            continue
+        await cache.remove(path)
+
+
+async def _background_drain(
+    cache: FileCacheMixin,
+    path: str,
+    it: CachableAsyncIterator,
+    max_bytes: int | None = None,
+    records: list[OpRecord] | None = None,
+) -> None:
+    """Drain an unconsumed stream and write to cache.
+
+    Cancelled by workspace.close() if the stream is still draining at
+    shutdown. If max_bytes is set and the drain exceeds it without
+    exhausting the source, the partial buffer is discarded and the path
+    is not cached (next read will fetch fresh from the resource).
+    The fingerprint is looked up after the drain: streaming backends
+    stamp their read record lazily, once the GET response arrives.
+    """
+    try:
+        if max_bytes is None:
+            materialized = await it.drain()
+            await cache.add(path,
+                            materialized,
+                            fingerprint=read_fingerprint(records, path))
+            return
+        materialized, fully_drained = await it.drain_bounded(max_bytes)
+        if fully_drained:
+            await cache.add(path,
+                            materialized,
+                            fingerprint=read_fingerprint(records, path))
+        else:
+            logger.info(
+                "cache drain budget exceeded for %s "
+                "(>%d bytes), skipping cache fill", path, max_bytes)
+    except asyncio.CancelledError:
+        logger.warning("background drain cancelled for %s", path)
+    except Exception:
+        logger.warning("background drain failed for %s", path, exc_info=True)

python/mirage/cache/file/mixin.py

@@ -0,0 +1,92 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+from collections.abc import Iterable
+
+
+class FileCacheMixin:
+    """LRU file cache mixin for resources.
+
+    Adds cache tracking (sizes, fingerprints, TTL, LRU order)
+    on top of any resource. Data lives in the resource's storage —
+    subclass implements the cache methods using resource's store.
+    """
+
+    async def get(self, key: str) -> bytes | None:
+        raise NotImplementedError
+
+    async def set(self,
+                  key: str,
+                  data: bytes,
+                  fingerprint: str | None = None,
+                  ttl: int | None = None) -> None:
+        raise NotImplementedError
+
+    async def add(self,
+                  key: str,
+                  data: bytes,
+                  fingerprint: str | None = None,
+                  ttl: int | None = None) -> bool:
+        raise NotImplementedError
+
+    async def remove(self, key: str) -> None:
+        raise NotImplementedError
+
+    async def exists(self, key: str) -> bool:
+        raise NotImplementedError
+
+    async def is_fresh(self, key: str, remote_fingerprint: str) -> bool:
+        raise NotImplementedError
+
+    async def clear(self) -> None:
+        raise NotImplementedError
+
+    def evict_paths(self, paths: Iterable[str]) -> None:
+        """Synchronously evict the given keys from the cache.
+
+        Load-time helper invoked from ``Workspace.load`` (sync) to drop
+        snapshot-restored bytes for paths the caller wants to re-fetch
+        live. Backends whose state cannot be mutated synchronously
+        (e.g. Redis) override this as a no-op.
+
+        Args:
+            paths (Iterable[str]): Cache keys to drop.
+        """
+        raise NotImplementedError
+
+    async def multi_get(self, keys: list[str]) -> list[bytes | None]:
+        return [await self.get(k) for k in keys]
+
+    async def multi_set(
+        self,
+        items: list[tuple[str, bytes]],
+        fingerprint: str | None = None,
+        ttl: int | None = None,
+    ) -> None:
+        for key, data in items:
+            await self.set(key, data, fingerprint=fingerprint, ttl=ttl)
+
+    @property
+    def cache_size(self) -> int | None:
+        """Cached bytes; None for stores that don't track them."""
+        raise NotImplementedError
+
+    @property
+    def cache_entries(self) -> int | None:
+        """Number of cached entries; None for stores that don't track them."""
+        return None
+
+    @property
+    def cache_limit(self) -> int:
+        raise NotImplementedError

python/mirage/cache/file/ram.py

@@ -0,0 +1,168 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+import time
+from collections import OrderedDict
+from collections.abc import Iterable
+
+from mirage.cache.file.entry import CacheEntry
+from mirage.cache.file.mixin import FileCacheMixin
+from mirage.cache.file.utils import default_fingerprint, parse_limit
+from mirage.cache.lock import KeyLockMixin
+from mirage.resource.ram import RAMResource
+
+
+class RAMFileCacheStore(RAMResource, FileCacheMixin, KeyLockMixin):
+    """RAMResource with LRU cache tracking.
+
+    Data lives in inherited _store.files (RAMStore).
+    _entries tracks LRU metadata only.
+    All RAM commands (cat, grep, head, ...) inherited.
+    """
+
+    def __init__(
+        self,
+        cache_limit: str | int = "512MB",
+        max_drain_bytes: int | None = None,
+    ) -> None:
+        super().__init__()
+        self._cache_limit: int = parse_limit(cache_limit)
+        self._cache_size: int = 0
+        self._entries: OrderedDict[str, CacheEntry] = OrderedDict()
+        self._drain_tasks: dict[str, asyncio.Task] = {}
+        self._clear_lock: asyncio.Lock = asyncio.Lock()
+        self.max_drain_bytes: int | None = max_drain_bytes
+
+    async def get(self, key: str) -> bytes | None:
+        async with self._lock_for(key):
+            entry = self._entries.get(key)
+            if entry is None:
+                return None
+            if entry.expired:
+                self._cache_size -= entry.size
+                del self._entries[key]
+                self._store.files.pop(key, None)
+                return None
+            self._entries.move_to_end(key)
+            return self._store.files.get(key)
+
+    async def set(self,
+                  key: str,
+                  data: bytes,
+                  fingerprint: str | None = None,
+                  ttl: int | None = None) -> None:
+        async with self._lock_for(key):
+            if key in self._entries:
+                self._cache_size -= self._entries[key].size
+                del self._entries[key]
+            if fingerprint is None:
+                fingerprint = default_fingerprint(data)
+            entry = CacheEntry(
+                size=len(data),
+                cached_at=int(time.time()),
+                fingerprint=fingerprint,
+                ttl=ttl,
+            )
+            self._entries[key] = entry
+            self._store.files[key] = data
+            self._cache_size += entry.size
+        await self._evict()
+
+    async def add(self,
+                  key: str,
+                  data: bytes,
+                  fingerprint: str | None = None,
+                  ttl: int | None = None) -> bool:
+        async with self._lock_for(key):
+            existing = self._entries.get(key)
+            if existing is not None and not existing.expired:
+                return False
+            if key in self._entries:
+                self._cache_size -= self._entries[key].size
+                del self._entries[key]
+            if fingerprint is None:
+                fingerprint = default_fingerprint(data)
+            entry = CacheEntry(
+                size=len(data),
+                cached_at=int(time.time()),
+                fingerprint=fingerprint,
+                ttl=ttl,
+            )
+            self._entries[key] = entry
+            self._store.files[key] = data
+            self._cache_size += entry.size
+        await self._evict()
+        return True
+
+    async def remove(self, key: str) -> None:
+        async with self._lock_for(key):
+            task = self._drain_tasks.pop(key, None)
+            if task:
+                task.cancel()
+            if key in self._entries:
+                self._cache_size -= self._entries[key].size
+                del self._entries[key]
+                self._store.files.pop(key, None)
+        self._discard_lock(key)
+
+    async def exists(self, key: str) -> bool:
+        entry = self._entries.get(key)
+        return entry is not None and not entry.expired
+
+    async def is_fresh(self, key: str, remote_fingerprint: str) -> bool:
+        entry = self._entries.get(key)
+        if entry is None:
+            return False
+        return entry.fingerprint == remote_fingerprint
+
+    async def clear(self) -> None:
+        async with self._clear_lock:
+            for task in self._drain_tasks.values():
+                task.cancel()
+            self._drain_tasks.clear()
+            self._entries.clear()
+            self._store.files.clear()
+            self._cache_size = 0
+            self._clear_locks()
+
+    def evict_paths(self, paths: Iterable[str]) -> None:
+        for key in paths:
+            entry = self._entries.pop(key, None)
+            if entry is not None:
+                self._cache_size -= entry.size
+            self._store.files.pop(key, None)
+
+    async def _evict(self) -> None:
+        while self._cache_size > self._cache_limit and self._entries:
+            evicted_key = next(iter(self._entries))
+            async with self._lock_for(evicted_key):
+                if evicted_key not in self._entries:
+                    continue
+                evicted = self._entries.pop(evicted_key)
+                self._cache_size -= evicted.size
+                self._store.files.pop(evicted_key, None)
+            self._discard_lock(evicted_key)
+
+    @property
+    def cache_size(self) -> int:
+        return self._cache_size
+
+    @property
+    def cache_entries(self) -> int:
+        return len(self._entries)
+
+    @property
+    def cache_limit(self) -> int:
+        return self._cache_limit

python/mirage/cache/file/redis.py

@@ -0,0 +1,130 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import asyncio
+from collections.abc import Iterable
+
+from mirage.cache.file.mixin import FileCacheMixin
+from mirage.cache.file.utils import default_fingerprint, parse_limit
+from mirage.resource.redis.redis import RedisResource
+
+
+class RedisFileCacheStore(RedisResource, FileCacheMixin):
+
+    def __init__(
+        self,
+        cache_limit: str | int = "512MB",
+        url: str = "redis://localhost:6379/0",
+        key_prefix: str = "mirage:cache:",
+        max_drain_bytes: int | None = None,
+    ) -> None:
+        super().__init__(url=url, key_prefix=key_prefix)
+        # Advisory only: unlike the RAM store there is no client-side
+        # LRU, so nothing evicts on overflow. Cap memory on the Redis
+        # server instead (maxmemory + maxmemory-policy allkeys-lru) to
+        # approximate the RAM store's eviction behavior.
+        self._cache_limit: int = parse_limit(cache_limit)
+        self._cache_client = self._store._client
+        self._data_prefix = f"{key_prefix}data:"
+        self._meta_prefix = f"{key_prefix}meta:"
+        self.max_drain_bytes: int | None = max_drain_bytes
+        self._drain_tasks: dict[str, asyncio.Task] = {}
+
+    async def get(self, key: str) -> bytes | None:
+        return await self._cache_client.get(f"{self._data_prefix}{key}")
+
+    async def set(
+        self,
+        key: str,
+        data: bytes,
+        fingerprint: str | None = None,
+        ttl: int | None = None,
+    ) -> None:
+        if fingerprint is None:
+            fingerprint = default_fingerprint(data)
+        pipe = self._cache_client.pipeline()
+        dk = f"{self._data_prefix}{key}"
+        mk = f"{self._meta_prefix}{key}"
+        pipe.set(dk, data)
+        pipe.set(mk, fingerprint)
+        if ttl is not None:
+            pipe.expire(dk, ttl)
+            pipe.expire(mk, ttl)
+        await pipe.execute()
+
+    async def add(
+        self,
+        key: str,
+        data: bytes,
+        fingerprint: str | None = None,
+        ttl: int | None = None,
+    ) -> bool:
+        dk = f"{self._data_prefix}{key}"
+        exists = await self._cache_client.exists(dk)
+        if exists:
+            return False
+        await self.set(key, data, fingerprint=fingerprint, ttl=ttl)
+        return True
+
+    async def remove(self, key: str) -> None:
+        task = self._drain_tasks.pop(key, None)
+        if task:
+            task.cancel()
+        pipe = self._cache_client.pipeline()
+        pipe.delete(f"{self._data_prefix}{key}")
+        pipe.delete(f"{self._meta_prefix}{key}")
+        await pipe.execute()
+
+    async def exists(self, key: str) -> bool:
+        return bool(await
+                    self._cache_client.exists(f"{self._data_prefix}{key}"))
+
+    async def is_fresh(self, key: str, remote_fingerprint: str) -> bool:
+        fp = await self._cache_client.get(f"{self._meta_prefix}{key}")
+        if fp is None:
+            return False
+        if isinstance(fp, bytes):
+            fp = fp.decode()
+        return fp == remote_fingerprint
+
+    async def clear(self) -> None:
+        for task in self._drain_tasks.values():
+            task.cancel()
+        self._drain_tasks.clear()
+        for pattern in (
+                f"{self._data_prefix}*",
+                f"{self._meta_prefix}*",
+        ):
+            keys: list = []
+            async for k in self._cache_client.scan_iter(pattern):
+                keys.append(k)
+            if keys:
+                await self._cache_client.delete(*keys)
+
+    def evict_paths(self, paths: Iterable[str]) -> None:
+        # No-op: Redis cache holds nothing restored from the snapshot
+        # (only RAM caches are repopulated by _restore_cache), and the
+        # snapshot load path is sync so we cannot await redis deletes
+        # here. If a caller needs to drop live Redis-cached entries, use
+        # await self.remove(key) per path from an async context.
+        pass
+
+    @property
+    def cache_size(self) -> int | None:
+        # Size lives in the redis server and is not tracked client-side.
+        return None
+
+    @property
+    def cache_limit(self) -> int:
+        return self._cache_limit

python/mirage/cache/file/utils.py

@@ -0,0 +1,29 @@
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
+
+import hashlib
+
+
+def parse_limit(limit: str | int) -> int:
+    if isinstance(limit, int):
+        return limit
+    s = limit.strip().upper()
+    for suffix, mult in [("GB", 1 << 30), ("MB", 1 << 20), ("KB", 1 << 10)]:
+        if s.endswith(suffix):
+            return int(s[:-len(suffix)]) * mult
+    return int(s)
+
+
+def default_fingerprint(data: bytes) -> str:
+    return hashlib.md5(data).hexdigest()