chore: import upstream snapshot with attribution

app/core/config.py

@@ -0,0 +1,121 @@
+import os
+import re
+import sys
+from pathlib import Path
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name, "").strip()
+    try:
+        return int(raw) if raw else default
+    except ValueError:
+        return default
+
+
+def _env_path(name: str, default: Path) -> Path:
+    raw = os.environ.get(name, "").strip()
+    return Path(raw).expanduser().resolve() if raw else default
+
+
+def _detect_device() -> str:
+    """Pick best available Torch device for Demucs. Override via
+    STEMDECK_DEMUCS_DEVICE env var ('cuda' | 'mps' | 'cpu'). Apple Silicon
+    silently falls back to CPU otherwise -- demucs's CLI default is
+    "cuda if available else cpu" and macOS has no CUDA, leaving the
+    integrated GPU idle and processing 3-5x slower than necessary."""
+    forced = os.environ.get("STEMDECK_DEMUCS_DEVICE", "").strip().lower()
+    if forced in ("cuda", "mps", "cpu"):
+        return forced
+    try:
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda"
+        if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
+            return "mps"
+    except ImportError:
+        pass
+    return "cpu"
+
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+STATIC_DIR = ROOT / "static"
+STEM_NAMES: tuple[str, ...] = ("vocals", "drums", "bass", "guitar", "piano", "other")
+JOB_ID_RE = re.compile(r"^[a-f0-9]{12}$")
+
+# Runtime knobs -- env-backed so Docker / desktop packaging / local dev can
+# tune without a code edit. STEMDECK_DATA_DIR is the portable app root for
+# mutable runtime data; when unset, dev behavior remains the repo-local jobs/
+# folder.
+PORTABLE_DATA_DIR_ENABLED = bool(os.environ.get("STEMDECK_DATA_DIR", "").strip())
+DATA_DIR = _env_path("STEMDECK_DATA_DIR", ROOT)
+JOBS_DIR = _env_path(
+    "STEMDECK_JOBS_DIR",
+    (DATA_DIR / "jobs") if PORTABLE_DATA_DIR_ENABLED else (ROOT / "jobs"),
+)
+CACHE_DIR = _env_path("STEMDECK_CACHE_DIR", DATA_DIR / "cache")
+DOWNLOADS_DIR = _env_path("STEMDECK_DOWNLOADS_DIR", DATA_DIR / "downloads")
+MODELS_DIR = _env_path("STEMDECK_MODELS_DIR", DATA_DIR / "models")
+LOGS_DIR = _env_path("STEMDECK_LOGS_DIR", DATA_DIR / "logs")
+FFMPEG_DIR = _env_path("STEMDECK_FFMPEG_DIR", DATA_DIR / "ffmpeg")
+FFMPEG_BIN = _env_path(
+    "STEMDECK_FFMPEG",
+    FFMPEG_DIR / ("ffmpeg.exe" if sys.platform.startswith("win") else "ffmpeg"),
+)
+FFPROBE_BIN = _env_path(
+    "STEMDECK_FFPROBE",
+    FFMPEG_DIR / ("ffprobe.exe" if sys.platform.startswith("win") else "ffprobe"),
+)
+DEMUCS_MODEL = os.environ.get("STEMDECK_DEMUCS_MODEL", "htdemucs_6s").strip() or "htdemucs_6s"
+DEMUCS_DEVICE = _detect_device()
+MAX_DURATION_SEC = max(60, _env_int("STEMDECK_MAX_DURATION_SEC", 1200))  # 20 min default
+JOB_TTL_SECONDS = max(300, _env_int("STEMDECK_JOB_TTL_SECONDS", 24 * 3600))  # 24 h default
+MAX_PENDING_JOBS = max(1, min(50, _env_int("STEMDECK_MAX_PENDING_JOBS", 3)))
+TIMEOUT_FFMPEG = _env_int("STEMDECK_TIMEOUT_FFMPEG", 300)
+TIMEOUT_ANALYZE = _env_int("STEMDECK_TIMEOUT_ANALYZE", 120)
+TIMEOUT_DEMUCS_STALL = _env_int("STEMDECK_TIMEOUT_DEMUCS_STALL", 1800)
+# Max height for the MP4 video stream pulled from YouTube (issue #219).
+# Capped to keep downloads reasonable; 1080p of a full song is large.
+VIDEO_MAX_HEIGHT = max(144, _env_int("STEMDECK_VIDEO_MAX_HEIGHT", 720))
+
+
+def ffmpeg_executable() -> str:
+    """Return the preferred FFmpeg executable.
+
+    In portable mode, setup places FFmpeg under DATA_DIR/ffmpeg. Prefer that
+    binary when present; otherwise fall back to PATH so local dev and Docker
+    keep working exactly as before.
+    """
+    return str(FFMPEG_BIN) if FFMPEG_BIN.is_file() else "ffmpeg"
+
+
+def ffprobe_executable() -> str:
+    """Return the preferred ffprobe executable (same bundled dir as ffmpeg)."""
+    return str(FFPROBE_BIN) if FFPROBE_BIN.is_file() else "ffprobe"
+
+
+def configure_portable_environment() -> None:
+    """Keep generated caches inside the portable data folder when requested.
+
+    This is intentionally best-effort. It only sets variables that are still
+    unset, so explicit caller/env choices win.
+    """
+    if FFMPEG_DIR.is_dir():
+        path = os.environ.get("PATH", "")
+        ffmpeg_path = str(FFMPEG_DIR)
+        if ffmpeg_path not in path.split(os.pathsep):
+            os.environ["PATH"] = ffmpeg_path + (os.pathsep + path if path else "")
+
+    if PORTABLE_DATA_DIR_ENABLED:
+        os.environ.setdefault("XDG_CACHE_HOME", str(CACHE_DIR))
+        os.environ.setdefault("TORCH_HOME", str(MODELS_DIR / "torch"))
+
+
+def ensure_runtime_dirs() -> None:
+    paths = (
+        (JOBS_DIR, CACHE_DIR, DOWNLOADS_DIR, MODELS_DIR, LOGS_DIR)
+        if PORTABLE_DATA_DIR_ENABLED
+        else (JOBS_DIR,)
+    )
+    for path in paths:
+        path.mkdir(parents=True, exist_ok=True)

app/core/models.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import dataclasses
+import time
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+
+class JobCancelled(Exception):
+    """Raised inside a pipeline stage when the job's cancel flag is set."""
+
+
+JobStatus = Literal[
+    "queued", "downloading", "analyzing", "separating", "processing", "done", "error", "cancelled"
+]
+
+
+def _set(job: Job, **fields: object) -> None:
+    """Mutate Job fields. SSE polling picks up the change automatically."""
+    for k, v in fields.items():
+        if k == "stage":
+            job.stage_message = v  # type: ignore[assignment]
+        else:
+            setattr(job, k, v)
+
+
+@dataclass
+class Job:
+    id: str
+    status: JobStatus = "queued"
+    progress: float = 0.0
+    stage_message: str = "Queued"
+    title: str | None = None
+    duration_sec: float | None = None
+    thumbnail: str | None = None
+    bpm: int | None = None
+    key: str | None = None
+    scale: str | None = None  # "Major" / "Natural Minor"
+    key_confidence: int | None = None  # 0-100 percent
+    lufs: float | None = None  # ITU-R BS.1770 integrated loudness (dB)
+    peak_db: float | None = None  # sample peak in dBFS (close to true peak)
+    dynamic_range: float | None = None  # peak_db - integrated LUFS (dB)
+    tempo_stability: int | None = None  # 0-100, beat interval consistency
+    stem_presence: dict[str, int] | None = None  # per-stem RMS 0-100
+    sections: list[dict] | None = None  # [{id, name, start, end, color}]
+    tags: list[str] | None = None  # YouTube tags + categories, lowercased, max 8
+    stems: list[dict[str, str]] = field(default_factory=list)
+    # Subset of stems the user chose at submit. The pipeline produces all
+    # 6 regardless (Demucs htdemucs_6s is fixed), but after collect we
+    # mix down only the selected ones into mix.wav so the user can
+    # download a single track containing just their chosen stems.
+    selected_stems: list[str] = field(default_factory=list)
+    mix_url: str | None = None  # populated when a strict subset was selected
+    source_url: str | None = None  # original URL or "local:<filename>" for file uploads
+    # True when a silent video track (video.mp4) was preserved from an .mp4
+    # upload, enabling the "Export Mix (with video)" MP4 export.
+    has_video: bool = False
+    error: str | None = None
+    # Set by POST /api/jobs/{id}/cancel; consumed by pipeline stages.
+    # Not surfaced via to_state() -- it's internal control state.
+    cancel_requested: bool = False
+    # Wall-clock timestamps for metadata-based sweep -- more predictable
+    # than directory mtime, which can be touched by unrelated FS events.
+    created_at: float = field(default_factory=time.time)
+
+    def to_state(self) -> dict[str, Any]:
+        return {
+            "job_id": self.id,
+            "status": self.status,
+            "progress": self.progress,
+            "stage": self.stage_message,
+            "title": self.title,
+            "duration": self.duration_sec,
+            "thumbnail": self.thumbnail,
+            "bpm": self.bpm,
+            "key": self.key,
+            "scale": self.scale,
+            "key_confidence": self.key_confidence,
+            "lufs": self.lufs,
+            "peak_db": self.peak_db,
+            "dynamic_range": self.dynamic_range,
+            "tempo_stability": self.tempo_stability,
+            "stem_presence": self.stem_presence,
+            "sections": self.sections,
+            "tags": self.tags,
+            "stems": self.stems,
+            "selected_stems": self.selected_stems,
+            "mix_url": self.mix_url,
+            "source_url": self.source_url,
+            "has_video": self.has_video,
+            "error": self.error,
+            "created_at": self.created_at,
+        }
+
+    def to_record(self) -> dict[str, Any]:
+        return {field: getattr(self, field) for field in _JOB_FIELDS}
+
+    @classmethod
+    def from_record(cls, data: dict[str, Any]) -> Job:
+        fields = {key: value for key, value in data.items() if key in _JOB_FIELDS}
+        job_id = str(fields.pop("id", "")).strip()
+        if not job_id:
+            raise ValueError("job record missing id")
+        job = cls(id=job_id)
+        for key, value in fields.items():
+            setattr(job, key, value)
+        job.cancel_requested = False
+        return job
+
+
+_JOB_FIELDS = frozenset(f.name for f in dataclasses.fields(Job) if f.name != "cancel_requested")

app/core/registry.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import json
+import logging
+import subprocess
+import threading
+from pathlib import Path
+
+from app.core.config import JOB_ID_RE, STEM_NAMES
+from app.core.models import Job
+
+logger = logging.getLogger("stemdeck.registry")
+
+REGISTRY_VERSION = 1
+
+_jobs: dict[str, Job] = {}
+# Active subprocesses keyed by job_id (currently only Demucs). Lets
+# POST /cancel terminate the running process from the API thread instead
+# of waiting for the pipeline thread to notice the cancel flag.
+_procs: dict[str, subprocess.Popen] = {}
+_lock = threading.Lock()
+_REGISTRY_FILE = "registry.json"
+_TERMINAL = {"done"}
+
+
+def register(job: Job) -> Job:
+    with _lock:
+        _jobs[job.id] = job
+    return job
+
+
+def register_if_capacity(job: Job, max_pending: int) -> bool:
+    """Atomically check pending count and register if under capacity.
+    Returns True if registered, False if the queue is full."""
+    with _lock:
+        pending = sum(1 for j in _jobs.values() if j.status == "queued")
+        if pending >= max_pending:
+            return False
+        _jobs[job.id] = job
+    return True
+
+
+def get(job_id: str) -> Job | None:
+    with _lock:
+        return _jobs.get(job_id)
+
+
+def remove(job_id: str) -> None:
+    with _lock:
+        _jobs.pop(job_id, None)
+        _procs.pop(job_id, None)
+
+
+def all_jobs() -> dict[str, Job]:
+    """Return a snapshot of the registry for sweep / cleanup."""
+    with _lock:
+        return dict(_jobs)
+
+
+def _migrate(data: dict) -> dict:
+    """Upgrade registry JSON to REGISTRY_VERSION incrementally.
+    Each block transforms v(n) → v(n+1) so older snapshots always catch up."""
+    version = data.get("version", 0)
+    if version < 1:
+        # v0 → v1: version field was absent; no structural change needed.
+        data["version"] = 1
+        version = 1
+    # Future migrations go here as `if version < N:` blocks.
+    return data
+
+
+def persist(jobs_dir: Path) -> None:
+    """Persist terminal jobs so completed library entries survive restarts."""
+    try:
+        jobs_dir.mkdir(parents=True, exist_ok=True)
+    except OSError:
+        logger.warning("cannot create jobs dir %s; skipping persist", jobs_dir, exc_info=True)
+        return
+    with _lock:
+        records = [
+            job.to_record()
+            for job in sorted(_jobs.values(), key=lambda item: item.created_at)
+            if job.status in _TERMINAL
+        ]
+        payload = json.dumps({"version": REGISTRY_VERSION, "jobs": records}, indent=2) + "\n"
+    path = jobs_dir / _REGISTRY_FILE
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(payload, encoding="utf-8")
+    tmp.replace(path)
+
+
+def restore(jobs_dir: Path) -> None:
+    """Load persisted jobs and recover completed orphan jobs from disk."""
+    jobs_dir.mkdir(parents=True, exist_ok=True)
+    path = jobs_dir / _REGISTRY_FILE
+    if path.is_file():
+        try:
+            data = _migrate(json.loads(path.read_text(encoding="utf-8")))
+            to_add = {}
+            for record in data.get("jobs", []):
+                job = Job.from_record(record)
+                if JOB_ID_RE.match(job.id) and job.status in _TERMINAL and job.title:
+                    to_add[job.id] = job
+            with _lock:
+                _jobs.update(to_add)
+        except (OSError, json.JSONDecodeError, TypeError, ValueError):
+            logger.warning("failed to load registry from %s", path, exc_info=True)
+
+    with _lock:
+        known = set(_jobs)
+    changed = False
+    for job_dir in jobs_dir.iterdir():
+        if not job_dir.is_dir() or not JOB_ID_RE.match(job_dir.name) or job_dir.name in known:
+            continue
+        recovered = _recover_done_job(job_dir)
+        if recovered is not None:
+            with _lock:
+                _jobs[recovered.id] = recovered
+            changed = True
+    if changed:
+        persist(jobs_dir)
+
+
+def _recover_done_job(job_dir: Path) -> Job | None:
+    stems_dir = job_dir / "stems"
+    if not stems_dir.is_dir():
+        return None
+    stems = [
+        {"name": name, "url": f"/api/jobs/{job_dir.name}/stems/{name}.wav"}
+        for name in ("original", *STEM_NAMES)
+        if (stems_dir / f"{name}.wav").is_file()
+    ]
+    if not stems:
+        return None
+    mix_url = None
+    if (stems_dir / "mix.wav").is_file():
+        mix_url = f"/api/jobs/{job_dir.name}/stems/mix.wav"
+    selected = [stem["name"] for stem in stems if stem["name"] in STEM_NAMES] or list(STEM_NAMES)
+    meta_path = job_dir / "metadata.json"
+    if not meta_path.is_file():
+        return None
+    meta: dict = {}
+    try:
+        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        pass
+    return Job(
+        id=job_dir.name,
+        status="done",
+        progress=1.0,
+        stage_message="Done",
+        stems=stems,
+        selected_stems=selected,
+        mix_url=mix_url,
+        created_at=job_dir.stat().st_mtime,
+        title=meta.get("title"),
+        thumbnail=meta.get("thumbnail"),
+        duration_sec=meta.get("duration_sec"),
+        bpm=meta.get("bpm"),
+        key=meta.get("key"),
+        scale=meta.get("scale"),
+        key_confidence=meta.get("key_confidence"),
+        lufs=meta.get("lufs"),
+        peak_db=meta.get("peak_db"),
+        dynamic_range=meta.get("dynamic_range"),
+        tempo_stability=meta.get("tempo_stability"),
+        stem_presence=meta.get("stem_presence"),
+        sections=meta.get("sections"),
+        tags=meta.get("tags"),
+    )
+
+
+def set_proc(job_id: str, proc: subprocess.Popen | None) -> None:
+    with _lock:
+        if proc is None:
+            _procs.pop(job_id, None)
+        else:
+            _procs[job_id] = proc
+
+
+def get_proc(job_id: str) -> subprocess.Popen | None:
+    with _lock:
+        return _procs.get(job_id)

app/core/settings.py

@@ -0,0 +1,140 @@
+"""Runtime, user-toggleable settings (persisted to disk).
+
+These are read live (unlike the env-var constants in config.py, which are fixed
+at startup), so the Settings UI can change them without a restart:
+
+- `allow_network`     — whether StemDeck answers requests from other devices.
+- `max_duration_sec`  — longest track accepted for processing.
+- `video_max_height`  — max video resolution for MP4 export / YouTube pulls.
+
+Defaults fall back to the config.py constants (which honor their env vars), so
+nothing changes until the user overrides a value.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+
+from app.core.config import DATA_DIR, MAX_DURATION_SEC, VIDEO_MAX_HEIGHT
+
+_log = logging.getLogger("stemdeck.settings")
+
+_SETTINGS_PATH = DATA_DIR / "settings.json"
+_LOCK = threading.RLock()
+_state: dict | None = None  # whole settings dict, loaded lazily
+
+# Clamp bounds. Max track length is capped at 20 min (the product ceiling).
+_DURATION_MIN, _DURATION_MAX = 60, 1200  # 1 min .. 20 min
+_HEIGHT_MIN, _HEIGHT_MAX = 144, 2160
+_PORT_MIN, _PORT_MAX = 1024, 65535
+DEFAULT_PORT = 8000
+
+
+def _default_allow_network() -> bool:
+    # STEMDECK_ALLOW_NETWORK takes precedence when set explicitly.
+    # Otherwise: desktop keeps network off (user opts in via UI toggle);
+    # server/Docker deployments open it by default since network access is
+    # the entire point of a headless deployment.
+    env = os.environ.get("STEMDECK_ALLOW_NETWORK")
+    if env is not None:
+        return env.strip() == "1"
+    return os.environ.get("STEMDECK_DESKTOP") != "1"
+
+
+def _load() -> dict:
+    try:
+        data = json.loads(_SETTINGS_PATH.read_text(encoding="utf-8"))
+        if isinstance(data, dict):
+            return data
+    except FileNotFoundError:
+        pass  # no settings file yet — first run; use defaults
+    except Exception:
+        # Corrupt/unreadable file: fall back to defaults rather than crash.
+        _log.warning("could not read settings from %s", _SETTINGS_PATH, exc_info=True)
+    return {}
+
+
+def _ensure() -> dict:
+    global _state
+    if _state is None:
+        _state = _load()
+    return _state
+
+
+def _save() -> None:
+    try:
+        _SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True)
+        _SETTINGS_PATH.write_text(json.dumps(_ensure()), encoding="utf-8")
+    except Exception:
+        # Persistence is best-effort (read-only FS, permissions): the in-memory
+        # value still applies for this session, so don't fail the request.
+        _log.warning("could not persist settings to %s", _SETTINGS_PATH, exc_info=True)
+
+
+def _num(v: object) -> int | None:
+    return int(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else None
+
+
+# ── allow_network ──
+def get_allow_network() -> bool:
+    with _LOCK:
+        v = _ensure().get("allow_network")
+        return v if isinstance(v, bool) else _default_allow_network()
+
+
+def set_allow_network(value: bool) -> bool:
+    with _LOCK:
+        _ensure()["allow_network"] = bool(value)
+        _save()
+        return bool(value)
+
+
+# ── max_duration_sec ──
+def get_max_duration_sec() -> int:
+    with _LOCK:
+        v = _num(_ensure().get("max_duration_sec"))
+        return max(_DURATION_MIN, min(_DURATION_MAX, v)) if v is not None else MAX_DURATION_SEC
+
+
+def set_max_duration_sec(value: int) -> int:
+    with _LOCK:
+        clamped = max(_DURATION_MIN, min(_DURATION_MAX, int(value)))
+        _ensure()["max_duration_sec"] = clamped
+        _save()
+        return clamped
+
+
+# ── video_max_height ──
+def get_video_max_height() -> int:
+    with _LOCK:
+        v = _num(_ensure().get("video_max_height"))
+        return max(_HEIGHT_MIN, min(_HEIGHT_MAX, v)) if v is not None else VIDEO_MAX_HEIGHT
+
+
+def set_video_max_height(value: int) -> int:
+    with _LOCK:
+        clamped = max(_HEIGHT_MIN, min(_HEIGHT_MAX, int(value)))
+        _ensure()["video_max_height"] = clamped
+        _save()
+        return clamped
+
+
+# ── port ──
+# The preferred port the server binds on launch. The desktop launcher reads this
+# (default 8000) before spawning the backend; a self-hosted server's --port wins.
+# Changing it needs a restart — the socket is bound at startup.
+def get_port() -> int:
+    with _LOCK:
+        v = _num(_ensure().get("port"))
+        return max(_PORT_MIN, min(_PORT_MAX, v)) if v is not None else DEFAULT_PORT
+
+
+def set_port(value: int) -> int:
+    with _LOCK:
+        clamped = max(_PORT_MIN, min(_PORT_MAX, int(value)))
+        _ensure()["port"] = clamped
+        _save()
+        return clamped