unclecode--crawl4ai/deploy/docker/artifacts.py

"""
artifacts.py - sandboxed, opaque-ID artifact store.

The screenshot/PDF endpoints used to accept a caller-supplied `output_path` and
write to it. `validate_output_path` was string-only (no realpath / O_NOFOLLOW),
so a symlink or a sibling-prefix name (".../outputs-evil") bypassed it and gave
an arbitrary write -> root RCE (e.g. /etc/cron.d) since /app was writable.

The fix: callers never name a path. The server owns the directory, the names,
and the bytes. Each artifact gets a server-generated opaque id; writes are
O_EXCL|O_NOFOLLOW 0600 into a 0700 dir off /tmp, size-capped and dir-quota'd;
retrieval validates a 32-hex id, lstat()s a non-symlink regular file, and
enforces a TTL. A janitor reaps expired/over-quota files.
"""

from __future__ import annotations

import os
import re
import time
import uuid
from typing import Dict, Tuple

ARTIFACT_DIR = os.environ.get("CRAWL4AI_ARTIFACT_DIR", "/var/lib/crawl4ai/outputs")
MAX_ARTIFACT_BYTES = int(os.environ.get("CRAWL4AI_MAX_ARTIFACT_BYTES", str(50 * 1024 * 1024)))
ARTIFACT_QUOTA_BYTES = int(os.environ.get("CRAWL4AI_ARTIFACT_QUOTA_BYTES", str(2 * 1024 * 1024 * 1024)))
ARTIFACT_TTL_SECONDS = int(os.environ.get("CRAWL4AI_ARTIFACT_TTL_SECONDS", "3600"))

_HEX32 = re.compile(r"^[0-9a-f]{32}$")
_KIND = {
    "png": (".png", "image/png"),
    "pdf": (".pdf", "application/pdf"),
}


class ArtifactError(Exception):
    pass


class ArtifactTooLarge(ArtifactError):
    pass


class QuotaExceeded(ArtifactError):
    pass


class ArtifactNotFound(ArtifactError):
    pass


def init_store() -> None:
    """Create the artifact dir 0700 (idempotent)."""
    os.makedirs(ARTIFACT_DIR, mode=0o700, exist_ok=True)
    try:
        os.chmod(ARTIFACT_DIR, 0o700)
    except OSError:
        pass


def _dir_size() -> int:
    total = 0
    try:
        with os.scandir(ARTIFACT_DIR) as it:
            for entry in it:
                try:
                    st = entry.stat(follow_symlinks=False)
                    if os.path.stat.S_ISREG(st.st_mode):
                        total += st.st_size
                except OSError:
                    continue
    except FileNotFoundError:
        return 0
    return total


def write_artifact(kind: str, data: bytes) -> Dict:
    """Write bytes under a server-generated opaque id. Returns metadata."""
    ext, mime = _KIND.get(kind, (".bin", "application/octet-stream"))
    if len(data) > MAX_ARTIFACT_BYTES:
        raise ArtifactTooLarge(f"artifact exceeds {MAX_ARTIFACT_BYTES} bytes")
    init_store()
    if _dir_size() + len(data) > ARTIFACT_QUOTA_BYTES:
        raise QuotaExceeded("artifact storage quota exceeded")

    artifact_id = uuid.uuid4().hex  # 32 hex chars, unguessable
    path = os.path.join(ARTIFACT_DIR, artifact_id + ext)
    # O_EXCL: never follow/overwrite an existing entry; O_NOFOLLOW: refuse a
    # symlink at the final component. Server-chosen name => no traversal.
    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
    fd = os.open(path, flags, 0o600)
    with os.fdopen(fd, "wb") as f:
        f.write(data)
    return {"artifact_id": artifact_id, "mime": mime, "size": len(data)}


def resolve_artifact(artifact_id: str) -> Tuple[str, str]:
    """Map an opaque id to (path, mime), enforcing hex-id, no-symlink, TTL.

    Raises ArtifactNotFound for an invalid id, a non-regular/symlink file, a
    missing file, or an expired artifact (so existence is never revealed).
    """
    if not isinstance(artifact_id, str) or not _HEX32.match(artifact_id):
        raise ArtifactNotFound()
    for ext, mime in _KIND.values():
        path = os.path.join(ARTIFACT_DIR, artifact_id + ext)
        try:
            st = os.lstat(path)  # do NOT follow symlinks
        except FileNotFoundError:
            continue
        if not os.path.stat.S_ISREG(st.st_mode):
            raise ArtifactNotFound()  # symlink / special file
        if time.time() - st.st_mtime > ARTIFACT_TTL_SECONDS:
            try:
                os.unlink(path)
            except OSError:
                pass
            raise ArtifactNotFound()
        return path, mime
    raise ArtifactNotFound()


def janitor() -> int:
    """Reap expired artifacts; if still over quota, reap oldest first."""
    init_store()
    now = time.time()
    reaped = 0
    entries = []
    try:
        with os.scandir(ARTIFACT_DIR) as it:
            for entry in it:
                try:
                    st = entry.stat(follow_symlinks=False)
                except OSError:
                    continue
                if not os.path.stat.S_ISREG(st.st_mode):
                    # remove anything that is not a regular file (e.g. a symlink)
                    try:
                        os.unlink(entry.path)
                        reaped += 1
                    except OSError:
                        pass
                    continue
                if now - st.st_mtime > ARTIFACT_TTL_SECONDS:
                    try:
                        os.unlink(entry.path)
                        reaped += 1
                    except OSError:
                        pass
                    continue
                entries.append((st.st_mtime, st.st_size, entry.path))
    except FileNotFoundError:
        return reaped

    total = sum(sz for _, sz, _ in entries)
    if total > ARTIFACT_QUOTA_BYTES:
        for _mtime, sz, path in sorted(entries):  # oldest first
            if total <= ARTIFACT_QUOTA_BYTES:
                break
            try:
                os.unlink(path)
                total -= sz
                reaped += 1
            except OSError:
                pass
    return reaped