strukto-ai--mirage/python/mirage/core/discord/readdir.py

# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========

import logging
from datetime import datetime, timedelta, timezone

import aiohttp

from mirage.accessor.discord import DiscordAccessor
from mirage.cache.index import IndexCacheStore, IndexEntry
from mirage.core.discord.channels import list_channels
from mirage.core.discord.entry import (DiscordResourceType, channel_entry,
                                       guild_entry, history_entry,
                                       member_entry, snowflake_to_date)
from mirage.core.discord.files import file_blob_name
from mirage.core.discord.guilds import list_guilds
from mirage.core.discord.history import list_messages_for_day
from mirage.core.discord.members import list_members
from mirage.types import PathSpec
from mirage.utils.errors import enoent
from mirage.utils.key_prefix import mount_key, mount_prefix_of

logger = logging.getLogger(__name__)

SOFT_HTTP_STATUSES = frozenset((403, 404, 429))


def _is_soft_error(exc: Exception) -> bool:
    return (isinstance(exc, aiohttp.ClientResponseError)
            and exc.status in SOFT_HTTP_STATUSES)


def _date_range(end_date: str, days: int = 30) -> list[str]:
    end = datetime.strptime(end_date, "%Y-%m-%d").date()
    return [(end - timedelta(days=i)).isoformat()
            for i in range(days - 1, -1, -1)]


def _normalize_path(path: PathSpec | str) -> tuple[str, str, str]:
    """Reduce input to (prefix, key, virtual_key)."""
    if isinstance(path, str):
        path = PathSpec(virtual=path,
                        directory=path,
                        resource_path=path.strip("/"))
    prefix = mount_prefix_of(path.virtual, path.resource_path)
    raw = path.directory if path.pattern else path.virtual
    if prefix and raw.startswith(prefix):
        raw = raw[len(prefix):] or "/"
    key = raw.strip("/")
    virtual_key = prefix + "/" + key if key else prefix or "/"
    return prefix, key, virtual_key


async def _readdir_root(
    accessor: DiscordAccessor,
    prefix: str,
    virtual_key: str,
    index: IndexCacheStore | None,
) -> list[str]:
    if index is not None:
        listing = await index.list_dir(virtual_key)
        if listing.entries is not None:
            return listing.entries
    guilds = await list_guilds(accessor.config)
    entries = []
    names = []
    for g in guilds:
        entry = guild_entry(g)
        entries.append((entry.vfs_name, entry))
        names.append(f"{prefix}/{entry.vfs_name}")
    if index is not None:
        await index.set_dir(virtual_key, entries)
    return names


async def _ensure_guild_id(
    accessor: DiscordAccessor,
    prefix: str,
    guild_part: str,
    index: IndexCacheStore | None,
    raw_path: str,
) -> str:
    if index is None:
        raise enoent(raw_path)
    guild_virtual_key = prefix + "/" + guild_part
    lookup = await index.get(guild_virtual_key)
    if lookup.entry is None:
        await _readdir_root(accessor, prefix, prefix or "/", index)
        lookup = await index.get(guild_virtual_key)
    if lookup.entry is None:
        raise enoent(raw_path)
    return lookup.entry.id


async def _readdir_guild_top(
    prefix: str,
    key: str,
) -> list[str]:
    return [f"{prefix}/{key}/channels", f"{prefix}/{key}/members"]


async def _readdir_channels(
    accessor: DiscordAccessor,
    prefix: str,
    key: str,
    virtual_key: str,
    parts: list[str],
    index: IndexCacheStore | None,
    raw_path: str,
) -> list[str]:
    if index is not None:
        listing = await index.list_dir(virtual_key)
        if listing.entries is not None:
            return listing.entries
    guild_id = await _ensure_guild_id(accessor, prefix, parts[0], index,
                                      raw_path)
    channels = await list_channels(accessor.config, guild_id)
    entries = []
    names = []
    for c in channels:
        entry = channel_entry(c)
        entries.append((entry.vfs_name, entry))
        names.append(f"{prefix}/{key}/{entry.vfs_name}")
    if index is not None:
        await index.set_dir(virtual_key, entries)
    return names


async def _readdir_members(
    accessor: DiscordAccessor,
    prefix: str,
    key: str,
    virtual_key: str,
    parts: list[str],
    index: IndexCacheStore | None,
    raw_path: str,
) -> list[str]:
    if index is not None:
        listing = await index.list_dir(virtual_key)
        if listing.entries is not None:
            return listing.entries
    guild_id = await _ensure_guild_id(accessor, prefix, parts[0], index,
                                      raw_path)
    members = await list_members(accessor.config, guild_id)
    entries = []
    names = []
    for m in members:
        entry = member_entry(m)
        entries.append((entry.vfs_name, entry))
        names.append(f"{prefix}/{key}/{entry.vfs_name}")
    if index is not None:
        await index.set_dir(virtual_key, entries)
    return names


async def _ensure_channel_lookup(
    accessor: DiscordAccessor,
    prefix: str,
    parts: list[str],
    index: IndexCacheStore,
    raw_path: str,
):
    channel_vk = f"{prefix}/{'/'.join(parts[:3])}"
    lookup = await index.get(channel_vk)
    if lookup.entry is None:
        await _readdir_channels(accessor, prefix, "/".join(parts[:2]),
                                f"{prefix}/{'/'.join(parts[:2])}", parts[:2],
                                index, raw_path)
        lookup = await index.get(channel_vk)
    if lookup.entry is None:
        raise enoent(raw_path)
    return lookup


async def _readdir_channel_dates(
    accessor: DiscordAccessor,
    prefix: str,
    key: str,
    virtual_key: str,
    parts: list[str],
    index: IndexCacheStore | None,
    raw_path: str,
) -> list[str]:
    if index is None:
        last_msg_id = ""
    else:
        listing = await index.list_dir(virtual_key)
        if listing.entries is not None:
            return listing.entries
        lookup = await _ensure_channel_lookup(accessor, prefix, parts, index,
                                              raw_path)
        last_msg_id = lookup.entry.remote_time
    if last_msg_id:
        end_date = snowflake_to_date(last_msg_id)
    else:
        end_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
    dates = _date_range(end_date)
    entries = []
    names = []
    for d in dates:
        entry = history_entry(key, d)
        entry.resource_type = DiscordResourceType.HISTORY
        entry.vfs_name = d
        entries.append((d, entry))
        names.append(f"{prefix}/{key}/{d}")
    if index is not None:
        await index.set_dir(virtual_key, entries)
    return names


async def _fetch_day(
    accessor: DiscordAccessor,
    channel_id: str,
    date_str: str,
    date_vkey: str,
    index: IndexCacheStore,
) -> None:
    """Walk the day's history once, populate date dir and files dir
    entries in the index. Tolerates soft HTTP errors (403/404/429) by
    sealing an empty date dir.
    """
    try:
        messages = await list_messages_for_day(accessor.config, channel_id,
                                               date_str)
    except aiohttp.ClientResponseError as e:
        if _is_soft_error(e):
            logger.debug("discord: history denied for %s/%s (%d); empty day",
                         channel_id, date_str, e.status)
            await index.set_dir(date_vkey, [])
            return
        raise
    chat_entry = IndexEntry(
        id=f"{channel_id}:{date_str}:chat",
        name="chat.jsonl",
        resource_type="discord/chat_jsonl",
        vfs_name="chat.jsonl",
    )
    files_entry = IndexEntry(
        id=f"{channel_id}:{date_str}:files",
        name="files",
        resource_type="discord/files_dir",
        vfs_name="files",
    )
    await index.set_dir(date_vkey, [
        ("chat.jsonl", chat_entry),
        ("files", files_entry),
    ])
    file_entries: list[tuple[str, IndexEntry]] = []
    for msg in messages:
        for att in msg.get("attachments") or []:
            if not att.get("id"):
                continue
            blob_name = file_blob_name(att)
            file_entries.append((blob_name,
                                 IndexEntry(
                                     id=str(att["id"]),
                                     name=att.get("filename") or "",
                                     resource_type="discord/file",
                                     vfs_name=blob_name,
                                     size=att.get("size"),
                                     extra={
                                         "url":
                                         att.get("url", ""),
                                         "proxy_url":
                                         att.get("proxy_url", ""),
                                         "content_type":
                                         att.get("content_type", ""),
                                         "message_id":
                                         msg.get("id", ""),
                                         "author":
                                         msg.get("author",
                                                 {}).get("username", ""),
                                         "channel_id":
                                         channel_id,
                                         "date":
                                         date_str,
                                     },
                                 )))
    await index.set_dir(f"{date_vkey}/files", file_entries)


async def _readdir_date_contents(
    accessor: DiscordAccessor,
    prefix: str,
    key: str,
    virtual_key: str,
    parts: list[str],
    index: IndexCacheStore | None,
    raw_path: str,
) -> list[str]:
    if index is None:
        raise enoent(raw_path)
    cached = await index.list_dir(virtual_key)
    if cached.entries is not None:
        return cached.entries
    lookup = await _ensure_channel_lookup(accessor, prefix, parts, index,
                                          raw_path)
    await _fetch_day(accessor, lookup.entry.id, parts[3], virtual_key, index)
    cached = await index.list_dir(virtual_key)
    if cached.entries is None:
        raise enoent(raw_path)
    return cached.entries


async def _readdir_files_dir(
    accessor: DiscordAccessor,
    prefix: str,
    key: str,
    virtual_key: str,
    parts: list[str],
    index: IndexCacheStore | None,
    raw_path: str,
) -> list[str]:
    if index is None:
        raise enoent(raw_path)
    cached = await index.list_dir(virtual_key)
    if cached.entries is not None:
        return cached.entries
    # Date dir lookup triggers _fetch_day which populates the files dir
    date_key = "/".join(parts[:4])
    date_vk = f"{prefix}/{date_key}"
    date_spec = PathSpec(virtual=date_vk,
                         directory=date_vk,
                         resource_path=mount_key(date_vk, prefix))
    await readdir(accessor, date_spec, index)
    cached = await index.list_dir(virtual_key)
    if cached.entries is None:
        raise enoent(raw_path)
    return cached.entries


async def readdir(
    accessor: DiscordAccessor,
    path: PathSpec,
    index: IndexCacheStore = None,
) -> list[str]:
    """List directory contents.

    Args:
        accessor (DiscordAccessor): discord accessor.
        path (PathSpec | str): resource-relative path.
        index (IndexCacheStore | None): index cache.
    """
    prefix, key, virtual_key = _normalize_path(path)
    raw_path = path.virtual if isinstance(path, PathSpec) else path

    if not key:
        return await _readdir_root(accessor, prefix, virtual_key, index)

    parts = key.split("/")

    if len(parts) == 1:
        if index is not None:
            lookup = await index.get(virtual_key)
            if lookup.entry is None:
                raise enoent(raw_path)
        return await _readdir_guild_top(prefix, key)

    if len(parts) == 2 and parts[1] == "channels":
        return await _readdir_channels(accessor, prefix, key, virtual_key,
                                       parts, index, raw_path)

    if len(parts) == 2 and parts[1] == "members":
        return await _readdir_members(accessor, prefix, key, virtual_key,
                                      parts, index, raw_path)

    if len(parts) == 3 and parts[1] == "channels":
        return await _readdir_channel_dates(accessor, prefix, key, virtual_key,
                                            parts, index, raw_path)

    if len(parts) == 4 and parts[1] == "channels":
        return await _readdir_date_contents(accessor, prefix, key, virtual_key,
                                            parts, index, raw_path)

    if (len(parts) == 5 and parts[1] == "channels" and parts[4] == "files"):
        return await _readdir_files_dir(accessor, prefix, key, virtual_key,
                                        parts, index, raw_path)

    return []


def is_dir_name(child: str) -> bool:
    # Entries are recognized by extension, so classification never needs
    # the stat fallback.
    name = child.rsplit("/", 1)[-1]
    return not (name.endswith(".json") or name.endswith(".jsonl"))