strukto-ai--mirage/python/mirage/core/s3/readdir.py

# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

import logging

from mirage.accessor.s3 import S3Accessor
from mirage.cache.index import IndexCacheStore, IndexEntry, ResourceType
from mirage.core.s3._client import (_client_kwargs, _prefix, _strip_prefix,
                                    async_session)
from mirage.core.s3.constants import SCOPE_ERROR
from mirage.core.timeutil import to_iso_z
from mirage.types import PathSpec
from mirage.utils.key_prefix import mount_prefix_of

logger = logging.getLogger(__name__)


async def readdir(accessor: S3Accessor, path: PathSpec,
                  index: IndexCacheStore) -> list[str]:
    if isinstance(path, str):
        path = PathSpec(virtual=path,
                        directory=path,
                        resource_path=path.strip("/"))
    if isinstance(path, PathSpec):
        prefix = mount_prefix_of(path.virtual, path.resource_path)
        # When called from resolve_glob with a pattern (e.g. *.txt),
        # use path.directory for the listing. Direct callers (ls, ops)
        # pass pattern=None so path.virtual is used.
        path = path.directory if path.pattern else path.virtual
    if prefix and path.startswith(prefix):
        rest = path[len(prefix):]
        if prefix.endswith("/") or rest == "" or rest.startswith("/"):
            path = rest or "/"
    config = accessor.config
    raw_key = prefix + path if prefix else path
    virtual_key = raw_key.rstrip("/") or "/"
    listing = await index.list_dir(virtual_key)
    if listing.entries is not None:
        return listing.entries
    pfx = _prefix(path, config)
    names: list[str] = []
    dir_keys: set[str] = set()
    sizes: dict[str, int | None] = {}
    times: dict[str, str] = {}
    session = async_session(config)
    async with session.client(**_client_kwargs(config)) as client:
        paginator = client.get_paginator("list_objects_v2")
        async for page in paginator.paginate(Bucket=config.bucket,
                                             Prefix=pfx,
                                             Delimiter="/"):
            for cp in page.get("CommonPrefixes") or []:
                child = cp["Prefix"].rstrip("/")
                if child:
                    key = "/" + _strip_prefix(child, config)
                    names.append(key)
                    dir_keys.add(key)
            for obj in page.get("Contents") or []:
                relative = obj["Key"][len(pfx):]
                if relative and "/" not in relative:
                    key = "/" + _strip_prefix(obj["Key"], config)
                    names.append(key)
                    sizes[key] = obj.get("Size")
                    last_mod = obj.get("LastModified")
                    times[key] = to_iso_z(last_mod) if last_mod else ""
    names = sorted(names)
    if len(names) > SCOPE_ERROR:
        logger.warning(
            "s3 readdir: %s returned %d entries (limit %d)",
            virtual_key,
            len(names),
            SCOPE_ERROR,
        )
    virtual_entries = sorted((prefix + e if prefix else e) for e in names)
    index_entries = []
    for e in names:
        name = e.rsplit("/", 1)[-1]
        if e in dir_keys:
            # S3 "folders" are synthetic common-prefixes with no object
            # of their own, so there is no LastModified or Size to record.
            entry = IndexEntry(id=e,
                               name=name,
                               resource_type=ResourceType.FOLDER)
        else:
            entry = IndexEntry(id=e,
                               name=name,
                               resource_type=ResourceType.FILE,
                               size=sizes.get(e),
                               remote_time=times.get(e, ""))
        index_entries.append((name, entry))
    await index.set_dir(virtual_key, index_entries)
    return virtual_entries