chore: import upstream snapshot with attribution

app/api/config.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.core.config import STEM_NAMES
+
+router = APIRouter()
+
+
+@router.get("/config")
+def get_config() -> dict:
+    return {"stem_names": list(STEM_NAMES)}

app/api/events.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import AsyncIterator
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
+
+from app.core.config import JOB_ID_RE
+from app.core.registry import get as registry_get
+
+router = APIRouter(tags=["events"])
+
+# Close SSE connections that outlive this threshold to prevent zombie
+# connections from accumulating when clients disconnect without a TCP RST.
+_MAX_SSE_SECONDS = 4 * 3600  # 4 hours
+# Hard cap on concurrent SSE connections to prevent resource exhaustion
+# from tab leaks or aggressive reconnect loops.
+_MAX_SSE_CONNECTIONS = 200
+# Counter is only mutated from the async event loop (no awaits between
+# check+increment), so no lock is needed.
+_sse_active = 0
+
+
+@router.get("/jobs/{job_id}/events")
+async def job_events(job_id: str) -> StreamingResponse:
+    """Server-Sent Events stream of job state updates. Closes when the job
+    reaches a terminal status (done, error, cancelled) or after 4 hours."""
+    global _sse_active
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    job = registry_get(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="job not found")
+    if _sse_active >= _MAX_SSE_CONNECTIONS:
+        raise HTTPException(status_code=503, detail="too many concurrent streams")
+    _sse_active += 1
+
+    async def stream() -> AsyncIterator[str]:
+        global _sse_active
+        try:
+            last = None
+            keepalive_at = 0
+            loop = asyncio.get_running_loop()
+            deadline = loop.time() + _MAX_SSE_SECONDS
+            while loop.time() < deadline:
+                snapshot = job.to_state()
+                serialized = json.dumps(snapshot)
+                if serialized != last:
+                    yield f"data: {serialized}\n\n"
+                    last = serialized
+                    keepalive_at = 0
+                if snapshot["status"] in ("done", "error", "cancelled"):
+                    return
+                keepalive_at += 1
+                if keepalive_at >= 75:  # ~15s
+                    yield ": keepalive\n\n"
+                    keepalive_at = 0
+                await asyncio.sleep(0.2)
+        finally:
+            _sse_active -= 1
+
+    return StreamingResponse(
+        stream(),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+    )

app/api/jobs.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import shutil
+import subprocess
+import uuid
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, field_validator
+
+from app.core.config import JOB_ID_RE, JOBS_DIR, MAX_PENDING_JOBS, STEM_NAMES, ffprobe_executable
+from app.core.models import Job
+from app.core.registry import all_jobs as registry_all_jobs
+from app.core.registry import get as registry_get
+from app.core.registry import get_proc as registry_get_proc
+from app.core.registry import persist as registry_persist
+from app.core.registry import register_if_capacity as registry_register_if_capacity
+from app.core.registry import remove as registry_remove
+from app.core.settings import get_max_duration_sec
+from app.pipeline import run_local_pipeline, run_pipeline
+from app.pipeline.download import InvalidYouTubeURL, validate_youtube_url
+
+router = APIRouter(tags=["jobs"])
+logger = logging.getLogger("stemdeck.api")
+
+_ALLOWED_EXTS = frozenset((".mp3", ".wav", ".flac", ".mp4", ".m4a"))
+_MAX_UPLOAD_BYTES = 400 * 1024 * 1024  # 400 MB
+_WS_RE = re.compile(r"\s+")
+
+
+def _sanitize_title(filename: str) -> str:
+    """Strip extension, normalize whitespace, cap at 120 chars."""
+    stem = Path(filename).stem
+    return _WS_RE.sub(" ", stem).strip()[:120]
+
+
+def _probe_duration(path: Path) -> float:
+    """Run ffprobe to get file duration in seconds."""
+    result = subprocess.run(
+        [
+            ffprobe_executable(),
+            "-v",
+            "quiet",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            str(path),
+        ],
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"ffprobe failed: {result.stderr.strip()}")
+    try:
+        return float(result.stdout.strip())
+    except ValueError as e:
+        raise RuntimeError(f"ffprobe returned non-numeric duration: {result.stdout!r}") from e
+
+
+def _check_file_size(file_obj: object) -> int:
+    """Seek to end, return size, rewind. Operates on the SpooledTemporaryFile
+    backing a starlette UploadFile — synchronous, suitable for to_thread."""
+    file_obj.seek(0, 2)  # type: ignore[union-attr]
+    size = file_obj.tell()  # type: ignore[union-attr]
+    file_obj.seek(0)  # type: ignore[union-attr]
+    return size
+
+
+def _copy_to_dest(src_file: object, dest: Path) -> None:
+    """Copy SpooledTemporaryFile contents to dest. Synchronous, run in thread."""
+    with dest.open("wb") as out:
+        shutil.copyfileobj(src_file, out)  # type: ignore[arg-type]
+
+
+def _rmtree_job(job_id: str) -> None:
+    job_dir = JOBS_DIR / job_id
+    if not job_dir.is_dir():
+        return
+    try:
+        shutil.rmtree(job_dir)
+    except Exception:
+        logger.warning("failed to remove job dir %s", job_dir, exc_info=True)
+
+
+def _task_error_cb(task: asyncio.Task) -> None:
+    if task.cancelled():
+        return
+    exc = task.exception()
+    if exc is not None:
+        logger.error("pipeline task raised unhandled exception", exc_info=exc)
+
+
+class JobRequest(BaseModel):
+    url: str
+    # Subset of stems to include in the post-processing "selected mix"
+    # audio file. None = all 6 (no extra mix produced; would equal the
+    # original). Unknown stem names are dropped silently rather than
+    # rejected, so a future model with extra stems doesn't break older
+    # clients pinning the old set.
+    stems: list[str] | None = None
+
+
+@router.post("")
+async def create_job(request: Request) -> dict[str, str]:
+    """Submit a YouTube URL (JSON body) or upload an audio file (multipart/form-data)
+    to start a stem-separation job. Returns the new job ID."""
+    ct = request.headers.get("content-type", "")
+    if "multipart/form-data" in ct:
+        return await _create_local_job(request)
+    return await _create_youtube_job(request)
+
+
+async def _create_youtube_job(request: Request) -> dict[str, str]:
+    try:
+        body = await request.json()
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=f"Invalid JSON: {e}") from e
+    try:
+        payload = JobRequest(**body)
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=str(e)) from e
+
+    try:
+        url = validate_youtube_url(payload.url)
+    except InvalidYouTubeURL as e:
+        raise HTTPException(status_code=422, detail=str(e)) from e
+
+    selected = [s for s in payload.stems if s in STEM_NAMES] if payload.stems else list(STEM_NAMES)
+    if not selected:
+        selected = list(STEM_NAMES)
+
+    job = Job(id=uuid.uuid4().hex[:12], selected_stems=selected, source_url=url)
+    if not registry_register_if_capacity(job, MAX_PENDING_JOBS):
+        raise HTTPException(status_code=503, detail="Server busy, please try again later")
+    task = asyncio.create_task(run_pipeline(job, url, JOBS_DIR))
+    task.add_done_callback(_task_error_cb)
+    return {"job_id": job.id}
+
+
+async def _create_local_job(request: Request) -> dict[str, str]:
+    # Fast pre-check: if already at capacity, reject before touching disk.
+    # The real atomic check happens in register_if_capacity after the upload.
+    if sum(1 for j in registry_all_jobs().values() if j.status == "queued") >= MAX_PENDING_JOBS:
+        raise HTTPException(status_code=503, detail="Server busy, please try again later")
+
+    # Quick pre-check on Content-Length to fail fast for obviously oversized
+    # uploads without buffering the whole body first.
+    cl_header = request.headers.get("content-length")
+    if cl_header:
+        try:
+            if int(cl_header) > _MAX_UPLOAD_BYTES + 4096:
+                raise HTTPException(status_code=422, detail="File exceeds 400 MB limit")
+        except ValueError:
+            pass
+
+    form = await request.form()
+    upload = form.get("file")
+    stems_raw = form.get("stems", "[]")
+
+    if upload is None or not hasattr(upload, "filename"):
+        raise HTTPException(status_code=422, detail="No file provided")
+
+    filename: str = getattr(upload, "filename", "") or ""
+    ext = Path(filename).suffix.lower()
+    if ext not in _ALLOWED_EXTS:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Unsupported file type '{ext}': accepted formats are .mp3, .wav, .flac, .mp4, and .m4a",
+        )
+
+    # Validate stems list from form field
+    try:
+        stems_list = json.loads(stems_raw)
+        if not isinstance(stems_list, list):
+            raise ValueError
+    except (json.JSONDecodeError, ValueError):
+        stems_list = []
+    selected = [s for s in stems_list if s in STEM_NAMES] or list(STEM_NAMES)
+
+    # Check actual file size (SpooledTemporaryFile is already buffered at this
+    # point; seek/tell are fast and don't re-read the body).
+    file_obj = upload.file  # type: ignore[union-attr]
+    file_size = await asyncio.to_thread(_check_file_size, file_obj)
+    if file_size == 0:
+        raise HTTPException(status_code=422, detail="Uploaded file is empty")
+    if file_size > _MAX_UPLOAD_BYTES:
+        raise HTTPException(status_code=422, detail="File exceeds 400 MB limit")
+
+    job_id = uuid.uuid4().hex[:12]
+    job_dir = JOBS_DIR / job_id
+    source_path = job_dir / f"source{ext}"
+
+    job_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        await asyncio.to_thread(_copy_to_dest, file_obj, source_path)
+
+        # Duration check before registering the job so a violation leaves no
+        # registered job and no leftover directory.
+        try:
+            duration = await asyncio.to_thread(_probe_duration, source_path)
+        except Exception as e:
+            raise HTTPException(status_code=422, detail=f"Could not read file duration: {e}") from e
+
+        max_duration = get_max_duration_sec()
+        if duration > max_duration:
+            raise HTTPException(
+                status_code=422,
+                detail=(f"File is {int(duration // 60)} min — limit is {max_duration // 60} min"),
+            )
+    except HTTPException:
+        shutil.rmtree(job_dir, ignore_errors=True)
+        raise
+
+    title = _sanitize_title(filename)
+    local_source_url = f"local:{title}"
+    job = Job(
+        id=job_id,
+        selected_stems=selected,
+        title=title,
+        duration_sec=duration,
+        source_url=local_source_url,
+    )
+    if not registry_register_if_capacity(job, MAX_PENDING_JOBS):
+        shutil.rmtree(job_dir, ignore_errors=True)
+        raise HTTPException(status_code=503, detail="Server busy, please try again later")
+    task = asyncio.create_task(run_local_pipeline(job, source_path, JOBS_DIR))
+    task.add_done_callback(_task_error_cb)
+    return {"job_id": job.id}
+
+
+@router.get("")
+def list_jobs() -> list[dict]:
+    """List all completed jobs in the library, sorted by creation time."""
+    return [
+        job.to_state()
+        for job in sorted(registry_all_jobs().values(), key=lambda j: j.created_at)
+        if job.status == "done"
+    ]
+
+
+@router.get("/{job_id}")
+def get_job(job_id: str) -> dict:
+    """Get the current state of a job by ID."""
+    job = registry_get(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="job not found")
+    return job.to_state()
+
+
+@router.post("/{job_id}/cancel")
+def cancel_job(job_id: str) -> dict:
+    """Request cancellation of a running job. Idempotent for terminal jobs."""
+    job = registry_get(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="job not found")
+    if job.status in ("done", "error", "cancelled"):
+        return job.to_state()
+    job.cancel_requested = True
+    proc = registry_get_proc(job_id)
+    if proc is not None and proc.poll() is None:
+        proc.terminate()
+    return job.to_state()
+
+
+_SECTION_ID_RE = re.compile(r"^[a-zA-Z0-9_\-]{1,64}$")
+_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{3,8}$")
+
+
+class SectionItem(BaseModel):
+    id: str
+    name: str
+    start: float
+    end: float
+    color: str
+
+    @field_validator("id")
+    @classmethod
+    def _check_id(cls, v: str) -> str:
+        if not _SECTION_ID_RE.match(v):
+            raise ValueError("invalid section id")
+        return v
+
+    @field_validator("name")
+    @classmethod
+    def _check_name(cls, v: str) -> str:
+        return v.strip()[:64] or "Section"
+
+    @field_validator("color")
+    @classmethod
+    def _check_color(cls, v: str) -> str:
+        if not _COLOR_RE.match(v):
+            raise ValueError("invalid color")
+        return v
+
+    @field_validator("start", "end")
+    @classmethod
+    def _check_time(cls, v: float) -> float:
+        if not (0 <= v < 86400):
+            raise ValueError("time out of range")
+        return round(v, 3)
+
+
+class SectionsBody(BaseModel):
+    sections: list[SectionItem]
+
+
+@router.patch("/{job_id}/sections")
+def update_sections(job_id: str, body: SectionsBody) -> dict:
+    """Save named timeline sections (intro, verse, chorus, etc.) for a done job."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    job = registry_get(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="job not found")
+
+    validated = [s.model_dump() for s in body.sections]
+    job.sections = validated
+
+    job_dir = (JOBS_DIR / job_id).resolve()
+    if not job_dir.is_relative_to(JOBS_DIR.resolve()):
+        raise HTTPException(status_code=404, detail="job not found")
+    meta_path = job_dir / "metadata.json"
+
+    meta: dict = {}
+    if meta_path.is_file():
+        try:
+            meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            pass
+    meta["sections"] = validated
+    try:
+        meta_path.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8")
+    except OSError as exc:
+        logger.exception("failed to write sections for %s: %s", job_id, exc)
+        raise HTTPException(status_code=500, detail="failed to save sections") from exc
+
+    registry_persist(JOBS_DIR)
+
+    return {"job_id": job_id, "sections": validated}
+
+
+@router.delete("/{job_id}")
+def delete_job(job_id: str) -> dict[str, str]:
+    """Delete a completed or failed job and remove its stem files from disk."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    job = registry_get(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="job not found")
+    if job.status not in ("done", "error", "cancelled"):
+        raise HTTPException(status_code=409, detail="job is still running")
+    _rmtree_job(job_id)
+    registry_remove(job_id)
+    registry_persist(JOBS_DIR)
+    return {"job_id": job_id, "status": "deleted"}

app/api/qr.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import io
+
+import segno
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import Response
+
+router = APIRouter()
+
+_MAX_LEN = 500
+
+
+@router.get("/qr")
+def get_qr(url: str) -> Response:
+    if not url or len(url) > _MAX_LEN:
+        raise HTTPException(status_code=422, detail="url required (max 500 chars)")
+    qr = segno.make_qr(url, error="m")
+    buf = io.BytesIO()
+    qr.save(buf, kind="svg", scale=5, border=2, dark="#1a1206", light="#ffffff")
+    return Response(
+        content=buf.getvalue(),
+        media_type="image/svg+xml",
+        headers={"Cache-Control": "public, max-age=3600"},
+    )

app/api/router.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.api.config import router as config_router
+from app.api.events import router as events_router
+from app.api.jobs import router as jobs_router
+from app.api.qr import router as qr_router
+from app.api.stems import router as stems_router
+
+router = APIRouter()
+router.include_router(config_router, tags=["config"])
+router.include_router(jobs_router, prefix="/jobs", tags=["jobs"])
+router.include_router(events_router, tags=["events"])
+router.include_router(stems_router, tags=["stems"])
+router.include_router(qr_router, tags=["qr"])

app/api/stems.py

@@ -0,0 +1,494 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+import subprocess
+import tempfile
+import uuid
+import zipfile
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import FileResponse, Response, StreamingResponse
+from starlette.background import BackgroundTask
+
+from app.core.config import JOB_ID_RE, JOBS_DIR, STEM_NAMES, TIMEOUT_FFMPEG, ffmpeg_executable
+from app.core.registry import get as registry_get
+
+logger = logging.getLogger("stemdeck.api")
+
+router = APIRouter(tags=["stems"])
+
+# Stem files served by this endpoint: the 6 demucs stems + two
+# pipeline-produced extras. "original" is the re-encoded source song
+# (added when the user picked a strict subset), "mix" is the ffmpeg
+# amix of the user's selected stems.
+_ALLOWED_NAMES = frozenset(STEM_NAMES) | {"original", "mix"}
+
+# Lanes the dynamic mixdown may sum: the 6 stems plus "original" (the complement
+# track shown when the user picked a subset). "mix" is excluded -- it is the
+# static pre-render this endpoint replaces. Gains are linear; the studio caps a
+# lane at 2.0, so this generous bound just rejects abusive values.
+_MIXDOWN_NAMES = frozenset(STEM_NAMES) | {"original"}
+_MIXDOWN_MAX_GAIN = 4.0
+
+# Output encoders by container/extension, shared by the dynamic mixdown and the
+# stems zip. WAV is lossless PCM, FLAC is lossless compressed, MP3 is VBR ~190 kbps.
+_ENCODE_ARGS = {
+    "wav": ["-c:a", "pcm_s16le"],
+    "mp3": ["-q:a", "2"],
+    "flac": ["-c:a", "flac"],
+}
+MIXDOWN_CODECS = {ext: [*args, "-f", ext] for ext, args in _ENCODE_ARGS.items()}
+MIXDOWN_MEDIA_TYPES = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac"}
+
+
+def _validate_stem_path(job_id: str, name: str):
+    """Shared guard: validate job_id, name, job state, and path. Returns resolved Path."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    if name not in _ALLOWED_NAMES:
+        raise HTTPException(status_code=404, detail="unknown stem")
+    job = registry_get(job_id)
+    if job is None or job.status != "done":
+        raise HTTPException(status_code=404, detail="job not ready")
+    path = (JOBS_DIR / job_id / "stems" / f"{name}.wav").resolve()
+    if not path.is_file() or not path.is_relative_to(JOBS_DIR.resolve()):
+        raise HTTPException(status_code=404, detail="stem not found")
+    return path
+
+
+def _parse_lane_gains(stems: str, gains: str) -> tuple[list[str], list[float]]:
+    """Parse and validate parallel comma-separated lane names and linear gains.
+    Shared by the audio mixdown and the MP4 video mux. Raises HTTPException
+    on malformed input, unknown lanes, or out-of-range gains."""
+    names = [s for s in stems.split(",") if s]
+    raw_gains = [g for g in gains.split(",") if g]
+    if not names or len(names) != len(raw_gains):
+        raise HTTPException(
+            status_code=422, detail="stems and gains must be non-empty and equal length"
+        )
+    try:
+        parsed_gains = [float(g) for g in raw_gains]
+    except ValueError:
+        raise HTTPException(status_code=422, detail="gains must be numbers") from None
+    if any(g < 0 or g > _MIXDOWN_MAX_GAIN for g in parsed_gains):
+        raise HTTPException(status_code=422, detail="gain out of range")
+    if not set(names) <= _MIXDOWN_NAMES:
+        raise HTTPException(status_code=422, detail="unknown stem requested")
+    return names, parsed_gains
+
+
+async def _stream_ffmpeg(cmd: list[str]):
+    """Yield ffmpeg stdout in 64 KB chunks; kill process on client disconnect."""
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.DEVNULL,
+    )
+    try:
+        while True:
+            chunk = await proc.stdout.read(65536)
+            if not chunk:
+                break
+            yield chunk
+    finally:
+        if proc.returncode is None:
+            proc.kill()
+        await proc.wait()
+
+
+async def _ensure_cached_mp3(src: Path) -> Path:
+    """Transcode `src` (a stem WAV) to a sibling `<name>.mp3`, cached on disk.
+    Re-encoding a full song on every request is the slow part of loading a track
+    on mobile (≈3s/stem × 6 in parallel); caching makes repeat loads instant.
+    Written atomically (temp + rename) so concurrent fetches can't serve a
+    partial file."""
+    dest = src.with_suffix(".mp3")
+    if dest.is_file() and dest.stat().st_mtime >= src.stat().st_mtime:
+        return dest
+    tmp = dest.with_name(f".{dest.name}.{uuid.uuid4().hex}.tmp")
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-y",
+        "-i",
+        str(src),
+        "-q:a",
+        "2",  # VBR ~190 kbps
+        "-f",
+        "mp3",
+        str(tmp),
+    ]
+    proc = await asyncio.create_subprocess_exec(
+        *cmd, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE
+    )
+    try:
+        _, stderr = await asyncio.wait_for(proc.communicate(), timeout=TIMEOUT_FFMPEG)
+    except (TimeoutError, asyncio.TimeoutError):
+        proc.kill()
+        await proc.wait()
+        tmp.unlink(missing_ok=True)
+        raise HTTPException(status_code=504, detail="mp3 transcode timed out") from None
+    if proc.returncode != 0:
+        tmp.unlink(missing_ok=True)
+        logger.warning(
+            "mp3 transcode failed for %s: %s", src.name, (stderr or b"").decode("utf-8", "replace")
+        )
+        raise HTTPException(status_code=500, detail="mp3 transcode failed")
+    os.replace(tmp, dest)
+    return dest
+
+
+@router.get("/jobs/{job_id}/stems/peaks.json")
+async def get_stem_peaks(job_id: str) -> Response:
+    """Return pre-computed waveform peaks for all stems."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    job = registry_get(job_id)
+    if job is None or job.status != "done":
+        raise HTTPException(status_code=404, detail="job not ready")
+    path = (JOBS_DIR / job_id / "stems" / "peaks.json").resolve()
+    if not path.is_file() or not path.is_relative_to(JOBS_DIR.resolve()):
+        raise HTTPException(status_code=404, detail="peaks not found")
+    return FileResponse(
+        path,
+        media_type="application/json",
+        headers={"Cache-Control": "public, max-age=31536000, immutable"},
+    )
+
+
+@router.api_route("/jobs/{job_id}/stems/{name}.wav", methods=["GET", "HEAD"], response_model=None)
+async def get_stem(
+    job_id: str,
+    name: str,
+    start: float | None = Query(default=None, ge=0, description="Trim start in seconds"),
+    end: float | None = Query(default=None, gt=0, description="Trim end in seconds"),
+) -> FileResponse | StreamingResponse:
+    """Download a WAV stem. Optional ?start=&end= trims to a time region."""
+    path = _validate_stem_path(job_id, name)
+
+    if start is None and end is None:
+        return FileResponse(path, media_type="audio/wav", filename=f"{name}.wav")
+
+    if start is None or end is None or start >= end:
+        raise HTTPException(
+            status_code=422,
+            detail="start and end are both required and start must be less than end",
+        )
+
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-ss",
+        str(start),
+        "-i",
+        str(path),
+        "-t",
+        str(end - start),
+        "-c:a",
+        "pcm_s16le",
+        "-f",
+        "wav",
+        "pipe:1",
+    ]
+    return StreamingResponse(
+        _stream_ffmpeg(cmd),
+        media_type="audio/wav",
+        headers={"Content-Disposition": f'attachment; filename="{name}_region.wav"'},
+    )
+
+
+@router.get("/jobs/{job_id}/stems/{name}.mp3")
+async def get_stem_mp3(
+    job_id: str,
+    name: str,
+    start: float | None = Query(default=None, ge=0, description="Trim start in seconds"),
+    end: float | None = Query(default=None, gt=0, description="Trim end in seconds"),
+) -> Response:
+    """Stem as MP3 (VBR ~190 kbps). Full stems are cached to disk; ?start=&end=
+    streams a freshly-trimmed region (uncached)."""
+    path = _validate_stem_path(job_id, name)
+
+    if (start is None) != (end is None) or (start is not None and start >= end):
+        raise HTTPException(
+            status_code=422,
+            detail="start and end are both required and start must be less than end",
+        )
+
+    # Full-stem requests (no trim) are cached to disk so repeat loads — the
+    # common case for the mobile player — are instant instead of re-encoding.
+    if start is None:
+        cached = await _ensure_cached_mp3(path)
+        return FileResponse(
+            cached,
+            media_type="audio/mpeg",
+            headers={
+                "Content-Disposition": f'attachment; filename="{name}.mp3"',
+                # Stems are immutable once a job is done — let the phone cache
+                # them so a re-load is instant and offline-friendly.
+                "Cache-Control": "public, max-age=31536000, immutable",
+            },
+        )
+
+    pre_seek = ["-ss", str(start)] if start is not None else []
+    post_seek = ["-t", str(end - start)] if start is not None else []
+
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        *pre_seek,
+        "-i",
+        str(path),
+        *post_seek,
+        "-q:a",
+        "2",  # VBR ~190 kbps
+        "-f",
+        "mp3",
+        "pipe:1",
+    ]
+    filename = f"{name}_region.mp3" if start is not None else f"{name}.mp3"
+    return StreamingResponse(
+        _stream_ffmpeg(cmd),
+        media_type="audio/mpeg",
+        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+    )
+
+
+@router.get("/jobs/{job_id}/mixdown.{ext}", response_model=None)
+async def get_mixdown(
+    job_id: str,
+    ext: str,
+    stems: str = Query(..., description="Comma-separated lane names to sum"),
+    gains: str = Query(..., description="Comma-separated linear gains, parallel to stems"),
+    start: float | None = Query(default=None, ge=0, description="Trim start in seconds"),
+    end: float | None = Query(default=None, gt=0, description="Trim end in seconds"),
+) -> StreamingResponse:
+    """Render a fresh mixdown of the given lanes at the given gains, streamed as
+    WAV or MP3. Mirrors the studio mixer (per-stem volume, mute, solo) so the
+    exported file matches what is heard. The master fader is intentionally not
+    applied -- it is a monitoring level, not part of the mix. Optional ?start=&end=
+    trims to a loop region."""
+    if ext not in ("wav", "mp3", "flac"):
+        raise HTTPException(status_code=404, detail="not found")
+
+    names, parsed_gains = _parse_lane_gains(stems, gains)
+    if (start is None) != (end is None) or (start is not None and start >= end):
+        raise HTTPException(
+            status_code=422,
+            detail="start and end are both required and start must be less than end",
+        )
+
+    # Validates job_id (404), job done (404), and path traversal (404) per stem.
+    paths = [_validate_stem_path(job_id, name) for name in names]
+
+    pre_seek = ["-ss", str(start)] if start is not None else []
+    post_seek = ["-t", str(end - start)] if start is not None else []
+
+    cmd: list[str] = [ffmpeg_executable(), "-nostdin", "-loglevel", "error"]
+    for p in paths:
+        cmd += [*pre_seek, "-i", str(p)]
+    # Apply each lane's gain, then sum with amix (normalize=0 keeps levels faithful,
+    # matching collect.py). A single audible lane skips amix (a 1-input amix is a no-op).
+    filters = [f"[{i}:a]volume={g:.6f}[a{i}]" for i, g in enumerate(parsed_gains)]
+    n = len(paths)
+    if n > 1:
+        labels = "".join(f"[a{i}]" for i in range(n))
+        filters.append(f"{labels}amix=inputs={n}:normalize=0[mix]")
+        out_label = "[mix]"
+    else:
+        out_label = "[a0]"
+    codec = MIXDOWN_CODECS[ext]
+    cmd += ["-filter_complex", ";".join(filters), "-map", out_label, *post_seek, *codec, "pipe:1"]
+
+    media_type = MIXDOWN_MEDIA_TYPES[ext]
+    return StreamingResponse(
+        _stream_ffmpeg(cmd),
+        media_type=media_type,
+        headers={"Content-Disposition": f'attachment; filename="mixdown.{ext}"'},
+    )
+
+
+def _safe_title(title: str | None) -> str:
+    """Sanitize a song title into a filename-safe slug (matches the frontend)."""
+    safe = re.sub(r"[^a-zA-Z0-9]+", "_", title or "")
+    safe = re.sub(r"_{2,}", "_", safe).strip("_")[:80].strip("_")
+    return safe or "stems"
+
+
+@router.get("/jobs/{job_id}/video.mp4", response_model=None)
+async def get_video_mixdown(
+    job_id: str,
+    stems: str = Query(..., description="Comma-separated lane names to sum"),
+    gains: str = Query(..., description="Comma-separated linear gains, parallel to stems"),
+) -> StreamingResponse:
+    """Mux a fresh audio mixdown of the current mixer state with the job's preserved
+    video into an MP4 (issue #219). Mirrors get_mixdown's audio graph (encoded
+    as AAC) and stream-copies video.mp4 -- the silent video kept from an .mp4 upload
+    or the real video stream downloaded for a YouTube job. 404 when the job has no
+    video (SoundCloud / plain audio uploads).
+
+    Streamed as fragmented MP4 (frag_keyframe+empty_moov) since the output pipe is
+    not seekable -- +faststart would require a seekable file. The full song is
+    exported; no region trim, to avoid A/V drift from stream-copy seeking."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    job = registry_get(job_id)
+    if job is None or job.status != "done":
+        raise HTTPException(status_code=404, detail="job not ready")
+
+    video_path = (JOBS_DIR / job_id / "video.mp4").resolve()
+    if not video_path.is_file() or not video_path.is_relative_to(JOBS_DIR.resolve()):
+        raise HTTPException(status_code=404, detail="no video track for this job")
+
+    names, parsed_gains = _parse_lane_gains(stems, gains)
+    # Validates job_id (404), job done (404), and path traversal (404) per stem.
+    paths = [_validate_stem_path(job_id, name) for name in names]
+
+    cmd: list[str] = [ffmpeg_executable(), "-nostdin", "-loglevel", "error"]
+    for p in paths:
+        cmd += ["-i", str(p)]
+    cmd += ["-i", str(video_path)]
+    video_idx = len(paths)
+    # Per-lane gain then amix (normalize=0 keeps levels faithful). A single audible
+    # lane skips amix (a 1-input amix is a no-op), matching get_mixdown.
+    filters = [f"[{i}:a]volume={g:.6f}[a{i}]" for i, g in enumerate(parsed_gains)]
+    n = len(paths)
+    if n > 1:
+        labels = "".join(f"[a{i}]" for i in range(n))
+        filters.append(f"{labels}amix=inputs={n}:normalize=0[mix]")
+        out_label = "[mix]"
+    else:
+        out_label = "[a0]"
+    cmd += [
+        "-filter_complex",
+        ";".join(filters),
+        "-map",
+        out_label,
+        "-map",
+        f"{video_idx}:v",
+        "-c:v",
+        "copy",
+        "-c:a",
+        "aac",
+        "-b:a",
+        "192k",
+        "-shortest",
+        "-movflags",
+        "frag_keyframe+empty_moov",
+        "-f",
+        "mp4",
+        "pipe:1",
+    ]
+
+    filename = f"{_safe_title(job.title)}_video.mp4"
+    return StreamingResponse(
+        _stream_ffmpeg(cmd),
+        media_type="video/mp4",
+        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+    )
+
+
+def _build_stems_zip(sources: list[tuple[str, Path]], fmt: str, dest: Path) -> None:
+    """Blocking: write the stems into a ZIP. WAV files are stored as-is; MP3 and
+    FLAC are transcoded per stem via ffmpeg. ZIP_STORED throughout - audio doesn't
+    meaningfully compress, and STORED keeps the build fast. Runs in a thread."""
+    if fmt == "wav":
+        with zipfile.ZipFile(dest, "w", zipfile.ZIP_STORED) as zf:
+            for name, p in sources:
+                zf.write(p, arcname=f"{name}.wav")
+        return
+    encode = _ENCODE_ARGS[fmt]
+    with tempfile.TemporaryDirectory() as td, zipfile.ZipFile(dest, "w", zipfile.ZIP_STORED) as zf:
+        for name, p in sources:
+            out = os.path.join(td, f"{name}.{fmt}")
+            cmd = [
+                ffmpeg_executable(),
+                "-nostdin",
+                "-loglevel",
+                "error",
+                "-i",
+                str(p),
+                *encode,
+                "-f",
+                fmt,
+                out,
+            ]
+            proc = subprocess.run(  # noqa: S603 — list args, no shell, trusted ffmpeg
+                cmd,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+                timeout=TIMEOUT_FFMPEG,
+            )
+            if proc.returncode != 0:
+                tail = proc.stderr[-2000:].decode("utf-8", "replace")
+                raise RuntimeError(f"ffmpeg failed for {name}: {tail}")
+            zf.write(out, arcname=f"{name}.{fmt}")
+
+
+@router.get("/jobs/{job_id}/stems/all.zip")
+async def get_all_stems_zip(
+    job_id: str,
+    fmt: str = Query(default="wav", alias="format"),
+    stems: str | None = Query(default=None, description="Comma-separated stems; default all"),
+) -> FileResponse:
+    """Bundle the requested stems into a single ZIP, named after the song.
+
+    `stems` is the active subset selected in the DAW (whitelisted). When omitted,
+    every available stem is included."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    if fmt not in ("wav", "mp3", "flac"):
+        raise HTTPException(status_code=422, detail="format must be 'wav', 'mp3', or 'flac'")
+    job = registry_get(job_id)
+    if job is None or job.status != "done":
+        raise HTTPException(status_code=404, detail="job not ready")
+
+    # Resolve the requested subset (whitelisted) or fall back to all stems.
+    if stems:
+        requested = {s for s in stems.split(",") if s}
+        if not requested <= set(STEM_NAMES):
+            raise HTTPException(status_code=422, detail="unknown stem requested")
+        wanted = [name for name in STEM_NAMES if name in requested]
+    else:
+        wanted = list(STEM_NAMES)
+
+    jobs_root = JOBS_DIR.resolve()
+    stems_dir = (JOBS_DIR / job_id / "stems").resolve()
+    if not stems_dir.is_dir() or not stems_dir.is_relative_to(jobs_root):
+        raise HTTPException(status_code=404, detail="stems not found")
+
+    sources: list[tuple[str, Path]] = []
+    for name in wanted:
+        p = (stems_dir / f"{name}.wav").resolve()
+        if p.is_file() and p.is_relative_to(jobs_root):
+            sources.append((name, p))
+    if not sources:
+        raise HTTPException(status_code=404, detail="no stems found")
+
+    fd, tmp = tempfile.mkstemp(prefix="stemdeck_zip_", suffix=".zip")
+    os.close(fd)
+    tmp_path = Path(tmp)
+    try:
+        await asyncio.to_thread(_build_stems_zip, sources, fmt, tmp_path)
+    except Exception:
+        tmp_path.unlink(missing_ok=True)
+        logger.exception("failed to build stems zip for job %s", job_id)
+        raise HTTPException(status_code=500, detail="failed to build archive") from None
+
+    filename = f"{_safe_title(job.title)}_stems.zip"
+    return FileResponse(
+        tmp_path,
+        media_type="application/zip",
+        filename=filename,
+        background=BackgroundTask(lambda: tmp_path.unlink(missing_ok=True)),
+    )

app/core/config.py

@@ -0,0 +1,121 @@
+import os
+import re
+import sys
+from pathlib import Path
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name, "").strip()
+    try:
+        return int(raw) if raw else default
+    except ValueError:
+        return default
+
+
+def _env_path(name: str, default: Path) -> Path:
+    raw = os.environ.get(name, "").strip()
+    return Path(raw).expanduser().resolve() if raw else default
+
+
+def _detect_device() -> str:
+    """Pick best available Torch device for Demucs. Override via
+    STEMDECK_DEMUCS_DEVICE env var ('cuda' | 'mps' | 'cpu'). Apple Silicon
+    silently falls back to CPU otherwise -- demucs's CLI default is
+    "cuda if available else cpu" and macOS has no CUDA, leaving the
+    integrated GPU idle and processing 3-5x slower than necessary."""
+    forced = os.environ.get("STEMDECK_DEMUCS_DEVICE", "").strip().lower()
+    if forced in ("cuda", "mps", "cpu"):
+        return forced
+    try:
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda"
+        if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
+            return "mps"
+    except ImportError:
+        pass
+    return "cpu"
+
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+STATIC_DIR = ROOT / "static"
+STEM_NAMES: tuple[str, ...] = ("vocals", "drums", "bass", "guitar", "piano", "other")
+JOB_ID_RE = re.compile(r"^[a-f0-9]{12}$")
+
+# Runtime knobs -- env-backed so Docker / desktop packaging / local dev can
+# tune without a code edit. STEMDECK_DATA_DIR is the portable app root for
+# mutable runtime data; when unset, dev behavior remains the repo-local jobs/
+# folder.
+PORTABLE_DATA_DIR_ENABLED = bool(os.environ.get("STEMDECK_DATA_DIR", "").strip())
+DATA_DIR = _env_path("STEMDECK_DATA_DIR", ROOT)
+JOBS_DIR = _env_path(
+    "STEMDECK_JOBS_DIR",
+    (DATA_DIR / "jobs") if PORTABLE_DATA_DIR_ENABLED else (ROOT / "jobs"),
+)
+CACHE_DIR = _env_path("STEMDECK_CACHE_DIR", DATA_DIR / "cache")
+DOWNLOADS_DIR = _env_path("STEMDECK_DOWNLOADS_DIR", DATA_DIR / "downloads")
+MODELS_DIR = _env_path("STEMDECK_MODELS_DIR", DATA_DIR / "models")
+LOGS_DIR = _env_path("STEMDECK_LOGS_DIR", DATA_DIR / "logs")
+FFMPEG_DIR = _env_path("STEMDECK_FFMPEG_DIR", DATA_DIR / "ffmpeg")
+FFMPEG_BIN = _env_path(
+    "STEMDECK_FFMPEG",
+    FFMPEG_DIR / ("ffmpeg.exe" if sys.platform.startswith("win") else "ffmpeg"),
+)
+FFPROBE_BIN = _env_path(
+    "STEMDECK_FFPROBE",
+    FFMPEG_DIR / ("ffprobe.exe" if sys.platform.startswith("win") else "ffprobe"),
+)
+DEMUCS_MODEL = os.environ.get("STEMDECK_DEMUCS_MODEL", "htdemucs_6s").strip() or "htdemucs_6s"
+DEMUCS_DEVICE = _detect_device()
+MAX_DURATION_SEC = max(60, _env_int("STEMDECK_MAX_DURATION_SEC", 1200))  # 20 min default
+JOB_TTL_SECONDS = max(300, _env_int("STEMDECK_JOB_TTL_SECONDS", 24 * 3600))  # 24 h default
+MAX_PENDING_JOBS = max(1, min(50, _env_int("STEMDECK_MAX_PENDING_JOBS", 3)))
+TIMEOUT_FFMPEG = _env_int("STEMDECK_TIMEOUT_FFMPEG", 300)
+TIMEOUT_ANALYZE = _env_int("STEMDECK_TIMEOUT_ANALYZE", 120)
+TIMEOUT_DEMUCS_STALL = _env_int("STEMDECK_TIMEOUT_DEMUCS_STALL", 1800)
+# Max height for the MP4 video stream pulled from YouTube (issue #219).
+# Capped to keep downloads reasonable; 1080p of a full song is large.
+VIDEO_MAX_HEIGHT = max(144, _env_int("STEMDECK_VIDEO_MAX_HEIGHT", 720))
+
+
+def ffmpeg_executable() -> str:
+    """Return the preferred FFmpeg executable.
+
+    In portable mode, setup places FFmpeg under DATA_DIR/ffmpeg. Prefer that
+    binary when present; otherwise fall back to PATH so local dev and Docker
+    keep working exactly as before.
+    """
+    return str(FFMPEG_BIN) if FFMPEG_BIN.is_file() else "ffmpeg"
+
+
+def ffprobe_executable() -> str:
+    """Return the preferred ffprobe executable (same bundled dir as ffmpeg)."""
+    return str(FFPROBE_BIN) if FFPROBE_BIN.is_file() else "ffprobe"
+
+
+def configure_portable_environment() -> None:
+    """Keep generated caches inside the portable data folder when requested.
+
+    This is intentionally best-effort. It only sets variables that are still
+    unset, so explicit caller/env choices win.
+    """
+    if FFMPEG_DIR.is_dir():
+        path = os.environ.get("PATH", "")
+        ffmpeg_path = str(FFMPEG_DIR)
+        if ffmpeg_path not in path.split(os.pathsep):
+            os.environ["PATH"] = ffmpeg_path + (os.pathsep + path if path else "")
+
+    if PORTABLE_DATA_DIR_ENABLED:
+        os.environ.setdefault("XDG_CACHE_HOME", str(CACHE_DIR))
+        os.environ.setdefault("TORCH_HOME", str(MODELS_DIR / "torch"))
+
+
+def ensure_runtime_dirs() -> None:
+    paths = (
+        (JOBS_DIR, CACHE_DIR, DOWNLOADS_DIR, MODELS_DIR, LOGS_DIR)
+        if PORTABLE_DATA_DIR_ENABLED
+        else (JOBS_DIR,)
+    )
+    for path in paths:
+        path.mkdir(parents=True, exist_ok=True)

app/core/models.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import dataclasses
+import time
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+
+class JobCancelled(Exception):
+    """Raised inside a pipeline stage when the job's cancel flag is set."""
+
+
+JobStatus = Literal[
+    "queued", "downloading", "analyzing", "separating", "processing", "done", "error", "cancelled"
+]
+
+
+def _set(job: Job, **fields: object) -> None:
+    """Mutate Job fields. SSE polling picks up the change automatically."""
+    for k, v in fields.items():
+        if k == "stage":
+            job.stage_message = v  # type: ignore[assignment]
+        else:
+            setattr(job, k, v)
+
+
+@dataclass
+class Job:
+    id: str
+    status: JobStatus = "queued"
+    progress: float = 0.0
+    stage_message: str = "Queued"
+    title: str | None = None
+    duration_sec: float | None = None
+    thumbnail: str | None = None
+    bpm: int | None = None
+    key: str | None = None
+    scale: str | None = None  # "Major" / "Natural Minor"
+    key_confidence: int | None = None  # 0-100 percent
+    lufs: float | None = None  # ITU-R BS.1770 integrated loudness (dB)
+    peak_db: float | None = None  # sample peak in dBFS (close to true peak)
+    dynamic_range: float | None = None  # peak_db - integrated LUFS (dB)
+    tempo_stability: int | None = None  # 0-100, beat interval consistency
+    stem_presence: dict[str, int] | None = None  # per-stem RMS 0-100
+    sections: list[dict] | None = None  # [{id, name, start, end, color}]
+    tags: list[str] | None = None  # YouTube tags + categories, lowercased, max 8
+    stems: list[dict[str, str]] = field(default_factory=list)
+    # Subset of stems the user chose at submit. The pipeline produces all
+    # 6 regardless (Demucs htdemucs_6s is fixed), but after collect we
+    # mix down only the selected ones into mix.wav so the user can
+    # download a single track containing just their chosen stems.
+    selected_stems: list[str] = field(default_factory=list)
+    mix_url: str | None = None  # populated when a strict subset was selected
+    source_url: str | None = None  # original URL or "local:<filename>" for file uploads
+    # True when a silent video track (video.mp4) was preserved from an .mp4
+    # upload, enabling the "Export Mix (with video)" MP4 export.
+    has_video: bool = False
+    error: str | None = None
+    # Set by POST /api/jobs/{id}/cancel; consumed by pipeline stages.
+    # Not surfaced via to_state() -- it's internal control state.
+    cancel_requested: bool = False
+    # Wall-clock timestamps for metadata-based sweep -- more predictable
+    # than directory mtime, which can be touched by unrelated FS events.
+    created_at: float = field(default_factory=time.time)
+
+    def to_state(self) -> dict[str, Any]:
+        return {
+            "job_id": self.id,
+            "status": self.status,
+            "progress": self.progress,
+            "stage": self.stage_message,
+            "title": self.title,
+            "duration": self.duration_sec,
+            "thumbnail": self.thumbnail,
+            "bpm": self.bpm,
+            "key": self.key,
+            "scale": self.scale,
+            "key_confidence": self.key_confidence,
+            "lufs": self.lufs,
+            "peak_db": self.peak_db,
+            "dynamic_range": self.dynamic_range,
+            "tempo_stability": self.tempo_stability,
+            "stem_presence": self.stem_presence,
+            "sections": self.sections,
+            "tags": self.tags,
+            "stems": self.stems,
+            "selected_stems": self.selected_stems,
+            "mix_url": self.mix_url,
+            "source_url": self.source_url,
+            "has_video": self.has_video,
+            "error": self.error,
+            "created_at": self.created_at,
+        }
+
+    def to_record(self) -> dict[str, Any]:
+        return {field: getattr(self, field) for field in _JOB_FIELDS}
+
+    @classmethod
+    def from_record(cls, data: dict[str, Any]) -> Job:
+        fields = {key: value for key, value in data.items() if key in _JOB_FIELDS}
+        job_id = str(fields.pop("id", "")).strip()
+        if not job_id:
+            raise ValueError("job record missing id")
+        job = cls(id=job_id)
+        for key, value in fields.items():
+            setattr(job, key, value)
+        job.cancel_requested = False
+        return job
+
+
+_JOB_FIELDS = frozenset(f.name for f in dataclasses.fields(Job) if f.name != "cancel_requested")

app/core/registry.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import json
+import logging
+import subprocess
+import threading
+from pathlib import Path
+
+from app.core.config import JOB_ID_RE, STEM_NAMES
+from app.core.models import Job
+
+logger = logging.getLogger("stemdeck.registry")
+
+REGISTRY_VERSION = 1
+
+_jobs: dict[str, Job] = {}
+# Active subprocesses keyed by job_id (currently only Demucs). Lets
+# POST /cancel terminate the running process from the API thread instead
+# of waiting for the pipeline thread to notice the cancel flag.
+_procs: dict[str, subprocess.Popen] = {}
+_lock = threading.Lock()
+_REGISTRY_FILE = "registry.json"
+_TERMINAL = {"done"}
+
+
+def register(job: Job) -> Job:
+    with _lock:
+        _jobs[job.id] = job
+    return job
+
+
+def register_if_capacity(job: Job, max_pending: int) -> bool:
+    """Atomically check pending count and register if under capacity.
+    Returns True if registered, False if the queue is full."""
+    with _lock:
+        pending = sum(1 for j in _jobs.values() if j.status == "queued")
+        if pending >= max_pending:
+            return False
+        _jobs[job.id] = job
+    return True
+
+
+def get(job_id: str) -> Job | None:
+    with _lock:
+        return _jobs.get(job_id)
+
+
+def remove(job_id: str) -> None:
+    with _lock:
+        _jobs.pop(job_id, None)
+        _procs.pop(job_id, None)
+
+
+def all_jobs() -> dict[str, Job]:
+    """Return a snapshot of the registry for sweep / cleanup."""
+    with _lock:
+        return dict(_jobs)
+
+
+def _migrate(data: dict) -> dict:
+    """Upgrade registry JSON to REGISTRY_VERSION incrementally.
+    Each block transforms v(n) → v(n+1) so older snapshots always catch up."""
+    version = data.get("version", 0)
+    if version < 1:
+        # v0 → v1: version field was absent; no structural change needed.
+        data["version"] = 1
+        version = 1
+    # Future migrations go here as `if version < N:` blocks.
+    return data
+
+
+def persist(jobs_dir: Path) -> None:
+    """Persist terminal jobs so completed library entries survive restarts."""
+    try:
+        jobs_dir.mkdir(parents=True, exist_ok=True)
+    except OSError:
+        logger.warning("cannot create jobs dir %s; skipping persist", jobs_dir, exc_info=True)
+        return
+    with _lock:
+        records = [
+            job.to_record()
+            for job in sorted(_jobs.values(), key=lambda item: item.created_at)
+            if job.status in _TERMINAL
+        ]
+        payload = json.dumps({"version": REGISTRY_VERSION, "jobs": records}, indent=2) + "\n"
+    path = jobs_dir / _REGISTRY_FILE
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(payload, encoding="utf-8")
+    tmp.replace(path)
+
+
+def restore(jobs_dir: Path) -> None:
+    """Load persisted jobs and recover completed orphan jobs from disk."""
+    jobs_dir.mkdir(parents=True, exist_ok=True)
+    path = jobs_dir / _REGISTRY_FILE
+    if path.is_file():
+        try:
+            data = _migrate(json.loads(path.read_text(encoding="utf-8")))
+            to_add = {}
+            for record in data.get("jobs", []):
+                job = Job.from_record(record)
+                if JOB_ID_RE.match(job.id) and job.status in _TERMINAL and job.title:
+                    to_add[job.id] = job
+            with _lock:
+                _jobs.update(to_add)
+        except (OSError, json.JSONDecodeError, TypeError, ValueError):
+            logger.warning("failed to load registry from %s", path, exc_info=True)
+
+    with _lock:
+        known = set(_jobs)
+    changed = False
+    for job_dir in jobs_dir.iterdir():
+        if not job_dir.is_dir() or not JOB_ID_RE.match(job_dir.name) or job_dir.name in known:
+            continue
+        recovered = _recover_done_job(job_dir)
+        if recovered is not None:
+            with _lock:
+                _jobs[recovered.id] = recovered
+            changed = True
+    if changed:
+        persist(jobs_dir)
+
+
+def _recover_done_job(job_dir: Path) -> Job | None:
+    stems_dir = job_dir / "stems"
+    if not stems_dir.is_dir():
+        return None
+    stems = [
+        {"name": name, "url": f"/api/jobs/{job_dir.name}/stems/{name}.wav"}
+        for name in ("original", *STEM_NAMES)
+        if (stems_dir / f"{name}.wav").is_file()
+    ]
+    if not stems:
+        return None
+    mix_url = None
+    if (stems_dir / "mix.wav").is_file():
+        mix_url = f"/api/jobs/{job_dir.name}/stems/mix.wav"
+    selected = [stem["name"] for stem in stems if stem["name"] in STEM_NAMES] or list(STEM_NAMES)
+    meta_path = job_dir / "metadata.json"
+    if not meta_path.is_file():
+        return None
+    meta: dict = {}
+    try:
+        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        pass
+    return Job(
+        id=job_dir.name,
+        status="done",
+        progress=1.0,
+        stage_message="Done",
+        stems=stems,
+        selected_stems=selected,
+        mix_url=mix_url,
+        created_at=job_dir.stat().st_mtime,
+        title=meta.get("title"),
+        thumbnail=meta.get("thumbnail"),
+        duration_sec=meta.get("duration_sec"),
+        bpm=meta.get("bpm"),
+        key=meta.get("key"),
+        scale=meta.get("scale"),
+        key_confidence=meta.get("key_confidence"),
+        lufs=meta.get("lufs"),
+        peak_db=meta.get("peak_db"),
+        dynamic_range=meta.get("dynamic_range"),
+        tempo_stability=meta.get("tempo_stability"),
+        stem_presence=meta.get("stem_presence"),
+        sections=meta.get("sections"),
+        tags=meta.get("tags"),
+    )
+
+
+def set_proc(job_id: str, proc: subprocess.Popen | None) -> None:
+    with _lock:
+        if proc is None:
+            _procs.pop(job_id, None)
+        else:
+            _procs[job_id] = proc
+
+
+def get_proc(job_id: str) -> subprocess.Popen | None:
+    with _lock:
+        return _procs.get(job_id)

app/core/settings.py

@@ -0,0 +1,140 @@
+"""Runtime, user-toggleable settings (persisted to disk).
+
+These are read live (unlike the env-var constants in config.py, which are fixed
+at startup), so the Settings UI can change them without a restart:
+
+- `allow_network`     — whether StemDeck answers requests from other devices.
+- `max_duration_sec`  — longest track accepted for processing.
+- `video_max_height`  — max video resolution for MP4 export / YouTube pulls.
+
+Defaults fall back to the config.py constants (which honor their env vars), so
+nothing changes until the user overrides a value.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+
+from app.core.config import DATA_DIR, MAX_DURATION_SEC, VIDEO_MAX_HEIGHT
+
+_log = logging.getLogger("stemdeck.settings")
+
+_SETTINGS_PATH = DATA_DIR / "settings.json"
+_LOCK = threading.RLock()
+_state: dict | None = None  # whole settings dict, loaded lazily
+
+# Clamp bounds. Max track length is capped at 20 min (the product ceiling).
+_DURATION_MIN, _DURATION_MAX = 60, 1200  # 1 min .. 20 min
+_HEIGHT_MIN, _HEIGHT_MAX = 144, 2160
+_PORT_MIN, _PORT_MAX = 1024, 65535
+DEFAULT_PORT = 8000
+
+
+def _default_allow_network() -> bool:
+    # STEMDECK_ALLOW_NETWORK takes precedence when set explicitly.
+    # Otherwise: desktop keeps network off (user opts in via UI toggle);
+    # server/Docker deployments open it by default since network access is
+    # the entire point of a headless deployment.
+    env = os.environ.get("STEMDECK_ALLOW_NETWORK")
+    if env is not None:
+        return env.strip() == "1"
+    return os.environ.get("STEMDECK_DESKTOP") != "1"
+
+
+def _load() -> dict:
+    try:
+        data = json.loads(_SETTINGS_PATH.read_text(encoding="utf-8"))
+        if isinstance(data, dict):
+            return data
+    except FileNotFoundError:
+        pass  # no settings file yet — first run; use defaults
+    except Exception:
+        # Corrupt/unreadable file: fall back to defaults rather than crash.
+        _log.warning("could not read settings from %s", _SETTINGS_PATH, exc_info=True)
+    return {}
+
+
+def _ensure() -> dict:
+    global _state
+    if _state is None:
+        _state = _load()
+    return _state
+
+
+def _save() -> None:
+    try:
+        _SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True)
+        _SETTINGS_PATH.write_text(json.dumps(_ensure()), encoding="utf-8")
+    except Exception:
+        # Persistence is best-effort (read-only FS, permissions): the in-memory
+        # value still applies for this session, so don't fail the request.
+        _log.warning("could not persist settings to %s", _SETTINGS_PATH, exc_info=True)
+
+
+def _num(v: object) -> int | None:
+    return int(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else None
+
+
+# ── allow_network ──
+def get_allow_network() -> bool:
+    with _LOCK:
+        v = _ensure().get("allow_network")
+        return v if isinstance(v, bool) else _default_allow_network()
+
+
+def set_allow_network(value: bool) -> bool:
+    with _LOCK:
+        _ensure()["allow_network"] = bool(value)
+        _save()
+        return bool(value)
+
+
+# ── max_duration_sec ──
+def get_max_duration_sec() -> int:
+    with _LOCK:
+        v = _num(_ensure().get("max_duration_sec"))
+        return max(_DURATION_MIN, min(_DURATION_MAX, v)) if v is not None else MAX_DURATION_SEC
+
+
+def set_max_duration_sec(value: int) -> int:
+    with _LOCK:
+        clamped = max(_DURATION_MIN, min(_DURATION_MAX, int(value)))
+        _ensure()["max_duration_sec"] = clamped
+        _save()
+        return clamped
+
+
+# ── video_max_height ──
+def get_video_max_height() -> int:
+    with _LOCK:
+        v = _num(_ensure().get("video_max_height"))
+        return max(_HEIGHT_MIN, min(_HEIGHT_MAX, v)) if v is not None else VIDEO_MAX_HEIGHT
+
+
+def set_video_max_height(value: int) -> int:
+    with _LOCK:
+        clamped = max(_HEIGHT_MIN, min(_HEIGHT_MAX, int(value)))
+        _ensure()["video_max_height"] = clamped
+        _save()
+        return clamped
+
+
+# ── port ──
+# The preferred port the server binds on launch. The desktop launcher reads this
+# (default 8000) before spawning the backend; a self-hosted server's --port wins.
+# Changing it needs a restart — the socket is bound at startup.
+def get_port() -> int:
+    with _LOCK:
+        v = _num(_ensure().get("port"))
+        return max(_PORT_MIN, min(_PORT_MAX, v)) if v is not None else DEFAULT_PORT
+
+
+def set_port(value: int) -> int:
+    with _LOCK:
+        clamped = max(_PORT_MIN, min(_PORT_MAX, int(value)))
+        _ensure()["port"] = clamped
+        _save()
+        return clamped

app/main.py

@@ -0,0 +1,376 @@
+from __future__ import annotations
+
+import asyncio
+import ctypes
+import functools
+import logging
+import os
+import re
+import signal
+import socket
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as package_version
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import FileResponse, PlainTextResponse
+from fastapi.staticfiles import StaticFiles
+
+from app.api.router import router
+from app.core.config import (
+    DEMUCS_DEVICE,
+    DEMUCS_MODEL,
+    FFMPEG_BIN,
+    JOBS_DIR,
+    STATIC_DIR,
+    configure_portable_environment,
+    ensure_runtime_dirs,
+)
+from app.core.registry import restore as restore_registry
+from app.core.settings import (
+    get_allow_network,
+    get_max_duration_sec,
+    get_port,
+    get_video_max_height,
+    set_allow_network,
+    set_max_duration_sec,
+    set_port,
+    set_video_max_height,
+)
+from app.pipeline.collect import sweep_old_jobs
+
+# Show our INFO-level logs through uvicorn's root handler. Without this,
+# Python's default root level (WARNING) silently drops every
+# logger.info(...) call across the app, including the analyze
+# diagnostics ("chroma:", "key candidates:").
+logging.getLogger("stemdeck").setLevel(logging.INFO)
+logging.getLogger("stemdeck").info("demucs config: model=%s device=%s", DEMUCS_MODEL, DEMUCS_DEVICE)
+
+configure_portable_environment()
+
+# Pre-import librosa so the first job submission doesn't pay the 1-2 s
+# cost of numpy/scipy/numba lazy initialization. Adds ~1 s to server
+# boot in exchange for snappier first-job UX. Best-effort: if librosa
+# isn't installed, analyze() degrades gracefully on its own.
+try:
+    import librosa  # noqa: F401  -- intentional warm-up import
+except ImportError:
+    pass
+
+_log = logging.getLogger("stemdeck")
+
+
+def _process_exists(pid: int) -> bool:
+    if os.name != "nt":
+        try:
+            os.kill(pid, 0)
+        except ProcessLookupError:
+            return False
+        except PermissionError:
+            return True
+        return True
+
+    PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+    ERROR_INVALID_PARAMETER = 87
+    kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
+    if handle:
+        kernel32.CloseHandle(handle)
+        return True
+    return ctypes.get_last_error() != ERROR_INVALID_PARAMETER
+
+
+def app_version() -> str:
+    # Version is git-tag-derived via hatch-vcs (#169). Prefer installed package
+    # metadata (set at install/build from the tag); fall back to the generated
+    # app/_version.py for non-installed runs, then a dev placeholder.
+    try:
+        return package_version("stemdeck")
+    except PackageNotFoundError:
+        pass
+    try:
+        from app._version import __version__
+
+        return str(__version__)
+    except Exception:
+        return "0.0.0-dev"
+
+
+def _sweep_disabled() -> bool:
+    """The desktop app is a personal, user-curated library (folders + Trash),
+    with its track list persisted permanently in ~/Documents/StemDeck. The 24h
+    job TTL sweep -- a sensible disk-hygiene default for the shared server/Docker
+    deployment -- would wrongly purge stems the user kept, leaving orphaned
+    library entries that ask to "re-upload to restore".
+
+    So skip the sweep under the desktop shell (STEMDECK_DESKTOP=1), or when a
+    self-hosted deployment opts into a persistent library
+    (STEMDECK_PERSIST_LIBRARY=1 -- set by default in run.sh). The user manages
+    disk via Trash. Shared/Docker deployments that set neither keep the sweep."""
+    return (
+        os.environ.get("STEMDECK_DESKTOP") == "1"
+        or os.environ.get("STEMDECK_PERSIST_LIBRARY") == "1"
+    )
+
+
+async def _sweep_loop() -> None:
+    if _sweep_disabled():
+        _log.info("job TTL sweep disabled (persistent library; user-managed)")
+        return
+    while True:
+        try:
+            await asyncio.to_thread(sweep_old_jobs, JOBS_DIR)
+        except Exception:
+            _log.warning("sweep failed", exc_info=True)
+        await asyncio.sleep(3600)
+
+
+async def _desktop_parent_watchdog(parent_pid: int) -> None:
+    while True:
+        if not _process_exists(parent_pid):
+            _log.info("desktop parent process exited; stopping backend")
+            # Send SIGTERM to ourselves so uvicorn runs its shutdown sequence
+            # instead of bypassing cleanup with os._exit().
+            os.kill(os.getpid(), signal.SIGTERM)
+            return
+        await asyncio.sleep(1)
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    _background_tasks = set()
+    t = asyncio.create_task(_sweep_loop())
+    _background_tasks.add(t)
+    t.add_done_callback(_background_tasks.discard)
+    if os.environ.get("STEMDECK_DESKTOP") == "1":
+        parent_pid = os.environ.get("STEMDECK_PARENT_PID")
+        if parent_pid:
+            try:
+                parent_pid_int = int(parent_pid)
+            except ValueError:
+                _log.warning("invalid STEMDECK_PARENT_PID=%r", parent_pid)
+            else:
+                if parent_pid_int > 0 and parent_pid_int != os.getpid():
+                    wt = asyncio.create_task(_desktop_parent_watchdog(parent_pid_int))
+                    _background_tasks.add(wt)
+                    wt.add_done_callback(_background_tasks.discard)
+    yield
+
+
+# Phones hitting the self-hosted server URL get the mobile UI; everything
+# else (desktop browsers, and the Tauri webviews, which all report desktop
+# user-agents) gets the DAW. Tablets are intentionally treated as desktop —
+# the DAW layout is usable there. "Mobi" is the cross-browser marker for a
+# phone form factor (Chrome/Firefox/Safari all include it); the rest cover
+# vendors that don't.
+_MOBILE_UA_RE = re.compile(
+    r"Mobi|Android|iPhone|iPod|IEMobile|BlackBerry|Opera Mini", re.IGNORECASE
+)
+
+
+def _is_mobile_ua(user_agent: str) -> bool:
+    return bool(user_agent) and _MOBILE_UA_RE.search(user_agent) is not None
+
+
+app = FastAPI(
+    title="StemDeck",
+    description="Paste a YouTube URL or upload an audio file, get audio stems split into a DAW-style player.",
+    version=app_version(),
+    lifespan=lifespan,
+)
+
+
+@app.get("/", include_in_schema=False)
+def index(request: Request) -> FileResponse:
+    """Serve the mobile shell to phones, the DAW to everyone else. `?ui=mobile`
+    / `?ui=desktop` forces either one (handy for testing from a desktop). This
+    route is registered before the StaticFiles mount at "/", so it wins for the
+    bare path while the mount still serves every other asset."""
+    ui = request.query_params.get("ui")
+    if ui == "mobile":
+        mobile = True
+    elif ui == "desktop":
+        mobile = False
+    else:
+        mobile = _is_mobile_ua(request.headers.get("user-agent", ""))
+    page = "mobile/index.html" if mobile else "index.html"
+    return FileResponse(STATIC_DIR / page)
+
+
+@app.get("/health", include_in_schema=False)
+def health_root() -> dict[str, object]:
+    return health()
+
+
+@app.get("/api/health", tags=["health"])
+def health() -> dict[str, object]:
+    return {
+        "name": "StemDeck",
+        "status": "ok",
+        "version": app_version(),
+        "ffmpeg_configured": FFMPEG_BIN.is_file(),
+        "demucs_model": DEMUCS_MODEL,
+        "demucs_device": DEMUCS_DEVICE,
+    }
+
+
+def _is_lan_ipv4(ip: str) -> bool:
+    """A reachable IPv4 LAN address to show another device: not IPv6 (link-local
+    needs a zone index and won't work in a browser), not loopback, not the
+    169.254.x auto-config range."""
+    if ":" in ip:  # IPv6
+        return False
+    if _is_loopback(ip) or ip.startswith("169.254."):
+        return False
+    parts = ip.split(".")
+    return len(parts) == 4 and all(p.isdigit() for p in parts)
+
+
+def _settings_payload() -> dict[str, object]:
+    return {
+        "allow_network": get_allow_network(),
+        "max_duration_sec": get_max_duration_sec(),
+        "video_max_height": get_video_max_height(),
+        "port": get_port(),
+    }
+
+
+@app.get("/api/settings", tags=["settings"])
+def get_settings(request: Request) -> dict[str, object]:
+    # LAN addresses other devices can use — loopback excluded (only works on the
+    # host). The port is whatever this request came in on.
+    port = request.url.port or 8000
+    addresses = sorted(f"http://{ip}:{port}" for ip in _local_ips() if _is_lan_ipv4(ip))
+    return {**_settings_payload(), "lan_addresses": addresses}
+
+
+@app.post("/api/settings", tags=["settings"])
+async def update_settings(request: Request) -> dict[str, object]:
+    """Update runtime settings. Reachable from the host machine always; from a
+    LAN device only while network access is currently on (the gate below), so a
+    phone can't change settings once the owner turned access off."""
+    try:
+        body = await request.json()
+    except Exception:
+        body = {}
+    if "allow_network" in body:
+        set_allow_network(bool(body["allow_network"]))
+    for key, setter in (
+        ("max_duration_sec", set_max_duration_sec),
+        ("video_max_height", set_video_max_height),
+        ("port", set_port),
+    ):
+        if key in body:
+            try:
+                setter(int(body[key]))
+            except (TypeError, ValueError):
+                raise HTTPException(status_code=422, detail=f"{key} must be an integer") from None
+    return _settings_payload()
+
+
+# Content-Security-Policy. Defense-in-depth so an injected string in the webview
+# can't run script (and, in the desktop app, reach the exposed Tauri IPC) — #171.
+# script-src has no 'unsafe-inline'/'eval': all JS is same-origin modules and the
+# inline scripts/onclick were moved out. 'unsafe-inline' is allowed for *styles*
+# only (the UI sets many style attributes). Allowances:
+#   connect-src  -> same-origin API/SSE, the GitHub update check, Tauri IPC
+#   img-src https: -> remote YouTube/SoundCloud thumbnails
+#   style/font   -> the Google Fonts <link>
+_CSP = (
+    "default-src 'self'; "
+    "script-src 'self'; "
+    "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
+    "font-src 'self' https://fonts.gstatic.com data:; "
+    "img-src 'self' data: blob: https:; "
+    "media-src 'self' blob: data:; "
+    # data:/blob: are required by multitrack.js (it fetches a data: URI during
+    # track init); without them Multitrack.create throws and no audio loads
+    # (#186). They are inline/same-origin schemes, not network endpoints, so
+    # they add no exfiltration channel — script-src below stays locked.
+    "connect-src 'self' https://api.github.com ipc: http://ipc.localhost data: blob:; "
+    "object-src 'none'; base-uri 'self'; frame-ancestors 'none'; form-action 'self'"
+)
+
+
+# Force browsers to revalidate static assets on every request. Without
+# this the JS/CSS modules can stick in disk cache across server
+# restarts -- updated HTML loads against stale modules and the form
+# silently breaks. `must-revalidate` keeps 304s working (cheap) while
+# guaranteeing the latest mtime is honored.
+@app.middleware("http")
+async def security_and_cache_headers(request: Request, call_next):
+    response = await call_next(request)
+    response.headers["Content-Security-Policy"] = _CSP
+    if not request.url.path.startswith("/api"):
+        response.headers["Cache-Control"] = "no-cache, must-revalidate"
+    return response
+
+
+def _is_loopback(host: str | None) -> bool:
+    if not host:
+        return False
+    if host.startswith("::ffff:"):  # IPv4-mapped IPv6
+        host = host[7:]
+    return host in {"127.0.0.1", "::1", "localhost"} or host.startswith("127.")
+
+
+@functools.lru_cache(maxsize=1)
+def _local_ips() -> frozenset[str]:
+    """The machine's own interface IPs. Used so the host always reaches the app
+    even via its LAN address (e.g. 192.168.x.x), not just 127.0.0.1 — turning
+    network access off must never cut the host off from its own server."""
+    ips: set[str] = set()
+    try:
+        hostname = socket.gethostname()
+        for info in socket.getaddrinfo(hostname, None):
+            ips.add(info[4][0])
+    except Exception:
+        # Best-effort: name resolution can fail on odd hostnames/configs; we
+        # still try the outbound-socket probe below and fall back to loopback.
+        _log.debug("hostname IP enumeration failed", exc_info=True)
+    try:  # primary outbound IP, robust when the hostname doesn't resolve them all
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 80))
+        ips.add(s.getsockname()[0])
+        s.close()
+    except Exception:
+        # Best-effort: no default route / offline — just return what we have.
+        _log.debug("outbound IP probe failed", exc_info=True)
+    return frozenset(ips)
+
+
+def _is_host_request(host: str | None) -> bool:
+    """True when the request originates from the machine StemDeck runs on —
+    whether via loopback or one of its own interface addresses."""
+    if _is_loopback(host):
+        return True
+    if not host:
+        return False
+    h = host[7:] if host.startswith("::ffff:") else host
+    return h in _local_ips()
+
+
+# Network availability gate (Settings → "Make StemDeck available on your
+# network"). Added after the headers middleware so it is the OUTERMOST layer and
+# short-circuits before anything else. It NEVER stops the server — it only
+# refuses requests from OTHER devices when availability is off. The host machine
+# (loopback or its own LAN IP) is always served, so the app keeps working
+# locally regardless of this setting.
+@app.middleware("http")
+async def network_gate(request: Request, call_next):
+    if not get_allow_network():
+        client_host = request.client.host if request.client else None
+        if not _is_host_request(client_host):
+            return PlainTextResponse(
+                "StemDeck is not available on the network. Enable it in Settings on the host machine.",
+                status_code=403,
+            )
+    return await call_next(request)
+
+
+app.include_router(router, prefix="/api")
+app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="static")
+
+ensure_runtime_dirs()
+restore_registry(JOBS_DIR)

app/pipeline/__init__.py

@@ -0,0 +1,3 @@
+from app.pipeline.runner import run_local_pipeline, run_pipeline
+
+__all__ = ["run_pipeline", "run_local_pipeline"]

app/pipeline/analyze.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import logging
+import subprocess
+from pathlib import Path
+
+from app.core.config import JOBS_DIR, TIMEOUT_ANALYZE, ffmpeg_executable
+from app.core.models import Job, _set
+
+logger = logging.getLogger("stemdeck.analyze")
+
+# Albrecht-Shanahan key profiles, derived from a corpus of popular music
+# (Albrecht & Shanahan, 2013). Critically, the minor profile here weights
+# b7 high (3.48) and M7 low (0.81) — the opposite of Temperley/Kostka-Payne,
+# which were derived from Bach chorales and bias toward harmonic minor's
+# leading tone. Pop/rock uses natural minor: the b7 is the diatonic
+# seventh and rings out constantly (e.g. open D in "Come As You Are",
+# which is in E minor and uses D as the b7). Values rescaled so that the
+# tonic weight is ≈5 to match the prior code's magnitude.
+_MAJOR_PROFILE = (
+    5.47,
+    0.14,
+    2.55,
+    0.14,
+    3.15,
+    2.16,
+    0.37,
+    4.92,
+    0.21,
+    1.84,
+    0.18,
+    1.86,
+)
+_MINOR_PROFILE = (
+    5.06,
+    0.14,
+    2.42,
+    2.42,
+    0.35,
+    1.96,
+    0.35,
+    4.16,
+    2.53,
+    0.28,
+    2.67,
+    0.62,
+)
+_PITCHES = ("C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B")
+
+# When the best-major and best-minor scores are this close, we prefer
+# minor. Pop/rock has a strong minor-mode prior; the algorithm often
+# walks toward the relative major because of an ostinato bass note
+# (e.g. "Come As You Are" hammers the open D string in an E minor song),
+# and minor is the better default when the call is genuinely ambiguous.
+_MINOR_TIE_BREAK_FRAC = 0.05
+
+
+def _correlate(profile: tuple[float, ...], chroma: list[float], shift: int) -> float:
+    n = len(profile)
+    rotated = [chroma[(i + shift) % n] for i in range(n)]
+    mean_p = sum(profile) / n
+    mean_c = sum(rotated) / n
+    num = sum((profile[i] - mean_p) * (rotated[i] - mean_c) for i in range(n))
+    denom_p = sum((profile[i] - mean_p) ** 2 for i in range(n)) ** 0.5
+    denom_c = sum((rotated[i] - mean_c) ** 2 for i in range(n)) ** 0.5
+    if denom_p == 0 or denom_c == 0:
+        return 0.0
+    return num / (denom_p * denom_c)
+
+
+def _detect_key(chroma_mean: list[float]) -> tuple[str, str, int]:
+    """Find the best-matching key by combining profile correlation with
+    root prominence. The Pearson correlation alone is fooled by relative
+    keys whose diatonic notes happen to overlap with the song's loud
+    pitches but whose own tonic is weak (e.g. picking A minor for an
+    E-minor song because E is its 5th and D is its 4th). Weighting by
+    the candidate root's chroma value forces the algorithm to also
+    confirm 'is this proposed tonic actually loud in the audio?'.
+    Logs the chroma vector and top-5 candidates for diagnostics.
+
+    Returns (label, scale_name, confidence_pct).
+    - label:        e.g. "G# maj"
+    - scale_name:   "Major" or "Natural Minor"
+    - confidence_pct: 0-100, derived from the gap between the winning
+                    candidate and the runner-up, normalized so a clear
+                    win ranks high and a near-tie ranks low."""
+    raw: list[tuple[float, float, str, int]] = []  # (weighted, pearson, label, root_idx)
+    for shift in range(12):
+        root_strength = chroma_mean[shift]
+        pearson_maj = _correlate(_MAJOR_PROFILE, chroma_mean, shift)
+        pearson_min = _correlate(_MINOR_PROFILE, chroma_mean, shift)
+        # Multiplicative root weighting. Pearson can be negative; when
+        # it is, a low-chroma root makes things less negative (closer to
+        # zero), which is actually the desired ordering.
+        raw.append((pearson_maj * root_strength, pearson_maj, f"{_PITCHES[shift]} maj", shift))
+        raw.append((pearson_min * root_strength, pearson_min, f"{_PITCHES[shift]} min", shift))
+    raw.sort(key=lambda x: x[0], reverse=True)
+
+    # Diagnostic log: chroma profile + top 5 candidates with both raw
+    # and weighted scores. Lets us see what the algorithm is "hearing".
+    chroma_str = ", ".join(f"{_PITCHES[i]}={chroma_mean[i]:.3f}" for i in range(12))
+    top5_str = ", ".join(
+        f"{label}={weighted:+.3f}(p{pearson:+.2f}*r{chroma_mean[idx]:.2f})"
+        for weighted, pearson, label, idx in raw[:5]
+    )
+    logger.debug("chroma: %s", chroma_str)
+    logger.debug("key candidates (top 5): %s", top5_str)
+
+    # Pick best major and best minor for the tie-break, both by the
+    # weighted score.
+    best_maj = next(c for c in raw if c[2].endswith("maj"))
+    best_min = next(c for c in raw if c[2].endswith("min"))
+
+    gap = abs(best_maj[0] - best_min[0])
+    threshold = max(abs(best_maj[0]), abs(best_min[0])) * _MINOR_TIE_BREAK_FRAC
+    # Near-tie -> prefer minor (pop/rock prior); clear winner -> use it.
+    winner = (best_maj if best_maj[0] > best_min[0] else best_min) if gap > threshold else best_min
+
+    # Confidence: gap between the winner and the runner-up that *isn't*
+    # the relative major/minor of the winner (those will always be near-
+    # ties with the algorithm's profile-correlation approach, so they
+    # tell us nothing about real ambiguity). Normalize so a healthy 0.15
+    # gap = 100% confident; tiny gap = 0%.
+    runner_up = next(c for c in raw if c[2] != winner[2])
+    confidence_score = winner[0] - runner_up[0]
+    confidence_pct = max(0, min(100, round(confidence_score / 0.15 * 100)))
+
+    label = winner[2]
+    scale_name = "Major" if label.endswith("maj") else "Natural Minor"
+    return label, scale_name, confidence_pct
+
+
+def _measure_loudness(y: object, sr: int) -> tuple[float | None, float | None]:
+    """Compute integrated loudness (LUFS, BS.1770) and sample peak (dBFS)
+    of the loaded mono signal. Returns (lufs, peak_db); either may be
+    None on failure or silence. We use sample peak rather than oversampled
+    true peak -- the difference is typically <1 dB and not worth the 4x
+    resample cost for a display field."""
+    import numpy as np
+
+    if y is None or getattr(y, "size", 0) == 0:
+        return None, None
+
+    peak_lin = float(np.abs(y).max())
+    peak_db = 20.0 * float(np.log10(peak_lin)) if peak_lin > 1e-9 else None
+
+    lufs: float | None = None
+    try:
+        import pyloudnorm as pyln
+
+        meter = pyln.Meter(sr)  # BS.1770-4 with default 400ms blocks
+        lufs_raw = float(meter.integrated_loudness(y))
+        # pyloudnorm returns -inf for silence; surface as None instead so
+        # the frontend can hide the field rather than render "-inf LUFS".
+        if np.isfinite(lufs_raw):
+            lufs = lufs_raw
+    except (ImportError, ValueError) as e:
+        # ValueError fires if the clip is shorter than the gating window.
+        logger.warning("LUFS measurement failed: %s", e)
+    return lufs, peak_db
+
+
+def _load_audio_ffmpeg(
+    source: Path, sr: int = 22050, duration: float = 180.0
+) -> tuple[object, int] | None:
+    """Decode `source` to a mono float32 numpy array at `sr` via ffmpeg.
+    Bypasses librosa's deprecated audioread fallback (which fires a
+    FutureWarning on .webm/.m4a/.opus inputs because soundfile can't
+    read those directly). Returns (samples, sr) or None on failure."""
+    import numpy as np
+
+    # Defence in depth: even though `source` is constructed by the server
+    # (never user-typed), confirm it's a real file inside JOBS_DIR before
+    # handing it to a subprocess. Belt-and-suspenders against a future
+    # caller change that would let a path slip in from elsewhere.
+    resolved = source.resolve()
+    jobs_resolved = JOBS_DIR.resolve()
+    if not resolved.is_file():
+        logger.warning("analyze source is not a file: %s", source)
+        return None
+    if not resolved.is_relative_to(jobs_resolved):
+        logger.warning(
+            "analyze source escapes JOBS_DIR (%s not under %s)",
+            resolved,
+            jobs_resolved,
+        )
+        return None
+
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(resolved),
+        "-ac",
+        "1",  # mono
+        "-ar",
+        str(sr),  # resample
+        "-f",
+        "f32le",  # raw 32-bit float little-endian
+        "-t",
+        str(duration),  # cap input duration
+        "-",  # write to stdout
+    ]
+    try:
+        proc = subprocess.run(cmd, capture_output=True, check=True, timeout=TIMEOUT_ANALYZE)
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as e:
+        logger.warning("ffmpeg decode failed for %s: %s", source, e)
+        return None
+    y = np.frombuffer(proc.stdout, dtype=np.float32)
+    if y.size == 0:
+        return None
+    return y, sr
+
+
+def compute_stem_presence(stems_dir: Path, selected_stems: list[str]) -> dict[str, int]:
+    """Load each extracted stem WAV, compute mean absolute amplitude, normalize
+    to 0-100. Only the stems that were selected (and therefore extracted) are
+    measured; the rest are omitted from the returned dict."""
+    import numpy as np
+
+    result: dict[str, int] = {}
+    rms_values: dict[str, float] = {}
+
+    for name in selected_stems:
+        wav_path = stems_dir / f"{name}.wav"
+        if not wav_path.is_file():
+            continue
+        loaded = _load_audio_ffmpeg(wav_path, sr=22050, duration=180.0)
+        if loaded is None:
+            continue
+        y, _ = loaded
+        rms_values[name] = float(np.sqrt(np.mean(y**2)))
+
+    if not rms_values:
+        return result
+
+    max_rms = max(rms_values.values())
+    if max_rms < 1e-9:
+        return {name: 0 for name in rms_values}
+
+    for name, rms in rms_values.items():
+        result[name] = max(0, min(100, round(rms / max_rms * 100)))
+
+    return result
+
+
+def analyze(job: Job, source: Path) -> tuple[int | None, str | None]:
+    """Best-effort BPM and key detection. On failure, returns (None, None)
+    and leaves job fields untouched -- the chips stay as placeholders."""
+    logger.info("analyze: entering for job %s, source=%s", job.id, source)
+    _set(job, status="analyzing", progress=0.0, stage="Analyzing audio...")
+    try:
+        import librosa
+    except ImportError:
+        logger.warning("librosa not installed -- skipping BPM/key analysis")
+        return None, None
+
+    try:
+        # Analyse the first 180 s. Decode via ffmpeg directly into numpy
+        # to avoid librosa's deprecated audioread fallback for
+        # .webm/.m4a/.opus inputs.
+        loaded = _load_audio_ffmpeg(source, sr=22050, duration=180.0)
+        if loaded is None:
+            return None, None
+        y, sr = loaded
+
+        # Harmonic / percussive separation. Beat tracking sees a cleaner
+        # onset envelope on the percussive component; chroma sees a
+        # cleaner pitch profile on the harmonic component (no cymbal
+        # smear, no kick fundamentals leaking in).
+        y_harmonic, y_percussive = librosa.effects.hpss(y)
+
+        tempo_arr, beat_frames = librosa.beat.beat_track(y=y_percussive, sr=sr)
+        try:
+            tempo = float(tempo_arr[0])  # type: ignore[index]
+        except (TypeError, IndexError):
+            tempo = float(tempo_arr)
+        bpm = int(round(tempo)) if tempo > 0 else None
+
+        # chroma_cqt is constant-Q based — better pitch resolution than
+        # chroma_stft, especially in the bass register where the open
+        # strings of a guitar live.
+        chroma = librosa.feature.chroma_cqt(y=y_harmonic, sr=sr)
+        chroma_mean = chroma.mean(axis=1).tolist()
+        if any(chroma_mean):
+            key, scale, key_confidence = _detect_key(chroma_mean)
+        else:
+            key, scale, key_confidence = None, None, None
+
+        # LUFS / peak. Computed on the same 22 kHz mono buffer; this
+        # loses a few dB of accuracy vs full-sample-rate stereo, but
+        # it's good enough for a UI display and adds ~50 ms to analyze.
+        lufs, peak_db = _measure_loudness(y, sr)
+
+        dynamic_range: float | None = None
+        if lufs is not None and peak_db is not None:
+            dynamic_range = round(peak_db - lufs, 1)
+
+        # Beat interval coefficient of variation → stability 0-100.
+        # CV = std/mean of inter-beat intervals; CV=0 is perfectly metronomic.
+        tempo_stability: int | None = None
+        import numpy as np
+
+        beat_times = librosa.frames_to_time(beat_frames, sr=sr)
+        if len(beat_times) > 2:
+            intervals = np.diff(beat_times)
+            mean_iv = float(intervals.mean())
+            if mean_iv > 0:
+                cv = float(intervals.std() / mean_iv)
+                tempo_stability = max(0, min(100, round((1 - min(cv, 1)) * 100)))
+
+        _set(
+            job,
+            bpm=bpm,
+            key=key,
+            scale=scale,
+            key_confidence=key_confidence,
+            lufs=lufs,
+            peak_db=peak_db,
+            dynamic_range=dynamic_range,
+            tempo_stability=tempo_stability,
+            progress=1.0,
+            stage="Analysis complete",
+        )
+        return bpm, key
+    except Exception as e:
+        logger.exception("analyze failed for job %s", job.id)
+        _set(job, stage=f"Analysis skipped ({e})")
+        return None, None

app/pipeline/collect.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+from app.core.config import (
+    DEMUCS_MODEL,
+    JOB_TTL_SECONDS,
+    STEM_NAMES,
+    TIMEOUT_FFMPEG,
+    ffmpeg_executable,
+)
+from app.core.models import Job
+from app.core.registry import all_jobs as registry_all
+from app.core.registry import persist as registry_persist
+from app.core.registry import remove as registry_remove
+from app.core.registry import set_proc
+
+logger = logging.getLogger("stemdeck.collect")
+
+
+def _rmtree(path: Path) -> None:
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError:
+        pass
+    except Exception:
+        logger.warning("failed to remove %s", path, exc_info=True)
+
+
+def _run_ffmpeg(job: Job, cmd: list[str]) -> bool:
+    """Run an ffmpeg command, registering the subprocess with the job
+    registry so POST /api/jobs/{id}/cancel can terminate it. Returns
+    True on success, False on failure or external termination.
+
+    Without registering the proc, an in-flight ffmpeg amix would block
+    cancellation for up to its 300s timeout -- the cancel flag is set
+    but the runner can't see it until subprocess.run returns. With
+    set_proc, the cancel API can call proc.terminate() directly and
+    communicate() returns within ~1s with a non-zero returncode."""
+    proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+    set_proc(job.id, proc)
+    try:
+        try:
+            _, stderr = proc.communicate(timeout=TIMEOUT_FFMPEG)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.communicate()
+            logger.warning("ffmpeg timed out for job %s", job.id)
+            return False
+        if proc.returncode != 0:
+            tail = (stderr or b"").decode(errors="replace").splitlines()[-3:]
+            logger.warning(
+                "ffmpeg exit %s for job %s: %s",
+                proc.returncode,
+                job.id,
+                " | ".join(tail) or "(no stderr)",
+            )
+            return False
+        return True
+    finally:
+        set_proc(job.id, None)
+
+
+_TERMINAL = frozenset(("done", "error", "cancelled"))
+
+
+def collect(job: Job, stems_root: Path, job_dir: Path) -> list[str]:
+    """Move Demucs-emitted stems into the job's stems/ dir and clean up
+    the demucs intermediate dir. Does NOT delete the source download --
+    cleanup_source() is called by the runner after any post-processing
+    that needs to re-encode the source (e.g. building original.wav)."""
+    target_dir = job_dir / "stems"
+    target_dir.mkdir(exist_ok=True)
+    found: list[str] = []
+    for name in STEM_NAMES:
+        src = stems_root / f"{name}.wav"
+        if src.exists():
+            shutil.move(str(src), target_dir / f"{name}.wav")
+            found.append(name)
+    _rmtree(job_dir / DEMUCS_MODEL)
+    if not found:
+        raise RuntimeError("no stems produced by demucs")
+    return found
+
+
+def cleanup_source(job_dir: Path) -> None:
+    """Delete the source audio file. Called after collect AND after any
+    post-processing that re-encodes the source (make_original_track).
+    The source is 100-300 MB, so getting rid of it is the bulk of disk
+    reclaim per job; only the stems remain."""
+    for f in job_dir.glob("source.*"):
+        f.unlink(missing_ok=True)
+
+
+def make_original_track(job: Job, job_dir: Path, stems_dir: Path) -> Path | None:
+    """Build the "Original" backing track at stems/original.wav as the
+    sum of the stems the user did NOT select. This way the studio can
+    play (original + each selected stem) and reconstruct the full song
+    without doubling the selected stems -- which is what would happen
+    if "original" were the raw source download (drum hits in original
+    + isolated drums.wav = drums at 2x amplitude).
+
+    Skipped when the user kept all 6 stems (no complement to mix) or
+    when none of the unselected stem WAVs are on disk."""
+    unselected = [s for s in STEM_NAMES if s not in job.selected_stems]
+    inputs = [stems_dir / f"{name}.wav" for name in unselected]
+    inputs = [p for p in inputs if p.exists()]
+    if not inputs:
+        return None
+    out = stems_dir / "original.wav"
+    cmd: list[str] = [
+        ffmpeg_executable(),
+        "-y",
+        "-nostdin",
+        "-loglevel",
+        "error",
+    ]
+    for p in inputs:
+        cmd += ["-i", str(p)]
+    if len(inputs) == 1:
+        # Single complement stem -- copy as-is so we still produce a
+        # canonical mix.wav-shaped output without invoking amix on a
+        # 1-input graph (which is a no-op anyway).
+        cmd += ["-c:a", "pcm_s16le", str(out)]
+    else:
+        filter_inputs = "".join(f"[{i}:a]" for i in range(len(inputs)))
+        cmd += [
+            "-filter_complex",
+            f"{filter_inputs}amix=inputs={len(inputs)}:normalize=0",
+            "-c:a",
+            "pcm_s16le",
+            str(out),
+        ]
+    return out if _run_ffmpeg(job, cmd) else None
+
+
+def make_selected_mix(job: Job, stems_dir: Path, found: list[str]) -> Path | None:
+    """If the user picked a strict subset of stems at submit time,
+    sum those stems with ffmpeg amix into mix.wav. Returns the output
+    path on success, or None when there's nothing to mix.
+
+    Returns the existing single stem path (no ffmpeg) if exactly one
+    stem was selected -- copying it to mix.wav would be 30 MB of
+    duplicate data. The caller uses the returned path's name for the
+    download URL, so a single-stem selection points the Download Mix
+    button directly at the existing stem file.
+
+    amix normalize=0 keeps stem amplitudes as-is. Demucs separations
+    sum back to (close to) the original signal, so a 2-stem subset
+    fits comfortably below 0 dBFS without normalization headroom."""
+    selected = [s for s in job.selected_stems if s in found]
+    if not selected:
+        return None
+    if len(selected) == 1:
+        return stems_dir / f"{selected[0]}.wav"
+    inputs = [stems_dir / f"{name}.wav" for name in selected]
+    out = stems_dir / "mix.wav"
+    cmd: list[str] = [
+        ffmpeg_executable(),
+        "-y",
+        "-nostdin",
+        "-loglevel",
+        "error",
+    ]
+    for p in inputs:
+        cmd += ["-i", str(p)]
+    filter_inputs = "".join(f"[{i}:a]" for i in range(len(inputs)))
+    cmd += [
+        "-filter_complex",
+        f"{filter_inputs}amix=inputs={len(inputs)}:normalize=0",
+        "-c:a",
+        "pcm_s16le",
+        str(out),
+    ]
+    return out if _run_ffmpeg(job, cmd) else None
+
+
+_PEAK_POINTS = 1500  # matches OVERVIEW_WAVE_POINTS in player.js
+
+
+def compute_stem_peaks(stems_dir: Path, stem_names: list[str]) -> None:
+    """Compute and cache [min, max] waveform peaks for each stem.
+    Failure is non-fatal — missing peaks.json degrades to client-side decode."""
+    peaks: dict[str, list[list[float]]] = {}
+    for name in stem_names:
+        path = stems_dir / f"{name}.wav"
+        if not path.is_file():
+            continue
+        try:
+            data, _ = sf.read(path, dtype="float32", always_2d=True)
+            ch = data[:, 0]
+            n = len(ch)
+            if n == 0:
+                continue
+            chunk = max(1, n // _PEAK_POINTS)
+            result: list[list[float]] = []
+            for i in range(0, n, chunk):
+                block = ch[i : i + chunk]
+                result.append([float(np.min(block)), float(np.max(block))])
+            peaks[name] = result[:_PEAK_POINTS]
+        except Exception:
+            logger.warning("could not compute peaks for %s/%s", stems_dir.name, name, exc_info=True)
+
+    if not peaks:
+        return
+
+    try:
+        tmp = stems_dir / "peaks.json.tmp"
+        tmp.write_text(json.dumps(peaks), encoding="utf-8")
+        tmp.replace(stems_dir / "peaks.json")
+    except Exception:
+        logger.warning("could not write peaks.json for %s", stems_dir.name, exc_info=True)
+
+
+def sweep_old_jobs(jobs_dir: Path) -> None:
+    """Delete job directories older than JOB_TTL_SECONDS and remove them from
+    the in-memory registry. Called hourly from the background sweep loop
+    started at app startup.
+
+    Prefers Job.created_at over directory mtime (which can be touched by
+    unrelated filesystem events), and never deletes the directory of an
+    active (non-terminal) registered job even if its timestamp looks old.
+    Falls back to mtime for orphan directories left over from a previous
+    server run, since the registry is in-memory only."""
+    cutoff = time.time() - JOB_TTL_SECONDS
+    if not jobs_dir.is_dir():
+        return
+    jobs = registry_all()
+    removed = False
+    for d in jobs_dir.iterdir():
+        if not d.is_dir():
+            continue
+        job = jobs.get(d.name)
+        if job is not None:
+            if job.status not in _TERMINAL:
+                continue  # never delete an active job's working dir
+            if job.created_at >= cutoff:
+                continue
+        elif d.stat().st_mtime >= cutoff:
+            continue
+        _rmtree(d)
+        registry_remove(d.name)
+        removed = True
+    if removed:
+        registry_persist(jobs_dir)

app/pipeline/download.py

@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import logging
+import re
+import time
+import urllib.parse
+from pathlib import Path
+
+from yt_dlp import YoutubeDL
+
+from app.core.config import FFMPEG_DIR
+from app.core.models import Job, JobCancelled, _set
+from app.core.settings import get_max_duration_sec, get_video_max_height
+
+logger = logging.getLogger("stemdeck.download")
+
+_MAX_RETRIES = 3
+_RETRY_BACKOFF = (2, 4, 8)  # seconds between attempts
+
+# Errors worth retrying — transient network blips.
+_RETRIABLE = (
+    "connection reset",
+    "ssl",
+    "timed out",
+    "network is unreachable",
+    "temporary failure",
+    "unable to download",
+    "read timed out",
+    "remotedisconnected",
+    "broken pipe",
+    "connection refused",
+)
+
+# Errors that will never succeed on retry — reject immediately.
+_NON_RETRIABLE = (
+    "private video",
+    "video unavailable",
+    "has been removed",
+    "http error 404",
+    "http error 403",
+    "not available in your country",
+    "age-restricted",
+)
+
+
+def _is_retriable(exc: Exception) -> bool:
+    msg = str(exc).lower()
+    if any(s in msg for s in _NON_RETRIABLE):
+        return False
+    return any(s in msg for s in _RETRIABLE)
+
+
+_VIDEO_ID_RE = re.compile(r"^[A-Za-z0-9_-]{11}$")
+_YOUTUBE_HOSTS = frozenset(
+    (
+        "youtube.com",
+        "www.youtube.com",
+        "m.youtube.com",
+        "music.youtube.com",
+        "youtu.be",
+    )
+)
+# Note: on.soundcloud.com (the share shortener) is intentionally excluded — it
+# redirects to arbitrary targets, which is an SSRF vector once handed to yt-dlp
+# (#173). Users must paste the full soundcloud.com URL.
+_SOUNDCLOUD_HOSTS = frozenset(("soundcloud.com", "www.soundcloud.com"))
+_ALLOWED_HOSTS = _YOUTUBE_HOSTS | _SOUNDCLOUD_HOSTS
+
+# Restrict yt-dlp to the extractors we actually support. Crucially this excludes
+# the "generic" extractor, so even a URL that slips past host validation cannot
+# make yt-dlp fetch an arbitrary host/redirect target (#173).
+_ALLOWED_EXTRACTORS = ["youtube", "soundcloud"]
+
+
+class InvalidYouTubeURL(ValueError):
+    """Raised at the API boundary for URLs we won't hand to yt-dlp."""
+
+
+def validate_youtube_url(url: str) -> str:
+    """Reject anything that isn't an http(s) URL on a known supported host.
+    YouTube URLs are normalized to single-video form; SoundCloud URLs are
+    passed through as-is. Gives callers a clean 422 instead of a yt-dlp
+    extractor stack trace."""
+    if not isinstance(url, str) or not url.strip():
+        raise InvalidYouTubeURL("URL is required")
+    url = url.strip()
+    try:
+        parsed = urllib.parse.urlparse(url)
+    except Exception as e:
+        raise InvalidYouTubeURL(f"could not parse URL: {e}") from e
+    if parsed.scheme not in ("http", "https"):
+        raise InvalidYouTubeURL("URL must use http or https")
+    host = (parsed.hostname or "").lower()
+    if host not in _ALLOWED_HOSTS:
+        raise InvalidYouTubeURL(f"unsupported host: {host or '(empty)'}")
+
+    if host in _SOUNDCLOUD_HOSTS:
+        return url
+
+    normalized = normalize_youtube_url(url)
+    # normalize_youtube_url returns the original on playlist-only URLs with
+    # no derivable seed video. We always expect the canonical watch?v=... form.
+    if not normalized.startswith("https://www.youtube.com/watch?v="):
+        raise InvalidYouTubeURL("could not extract a video ID from URL")
+    return normalized
+
+
+def normalize_youtube_url(url: str) -> str:
+    """Coerce a YouTube URL to a single-video form so yt-dlp doesn't end up in
+    the playlist extractor. Pass non-YouTube URLs through unchanged.
+
+    Cases handled:
+      * `watch?v=X&list=...` -> `watch?v=X` (drop the playlist context)
+      * `?list=RD<videoId>&start_radio=1` -> `watch?v=<videoId>` (Radio
+        playlists embed the seed in the list ID; YouTube refuses to view the
+        playlist directly with "This playlist type is unviewable.")
+      * `youtu.be/<videoId>` -> `watch?v=<videoId>`
+      * `youtube.com/shorts/<videoId>` -> `watch?v=<videoId>`
+    Everything else (PL/OL/algorithmic playlists with no derivable seed) is
+    left alone -- yt-dlp will surface its own error.
+    """
+    try:
+        parsed = urllib.parse.urlparse(url)
+    except Exception:
+        return url
+    host = (parsed.hostname or "").lower()
+    for prefix in ("www.", "m.", "music."):
+        if host.startswith(prefix):
+            host = host[len(prefix) :]
+            break
+    if host not in ("youtube.com", "youtu.be"):
+        return url
+
+    qs = urllib.parse.parse_qs(parsed.query)
+    if (vid := (qs.get("v") or [None])[0]) and _VIDEO_ID_RE.match(vid):
+        return f"https://www.youtube.com/watch?v={vid}"
+
+    if (
+        (lst := (qs.get("list") or [None])[0])
+        and lst.startswith("RD")
+        and _VIDEO_ID_RE.match(lst[2:13])
+    ):
+        return f"https://www.youtube.com/watch?v={lst[2:13]}"
+
+    if host == "youtu.be":
+        vid = parsed.path.lstrip("/")
+        if _VIDEO_ID_RE.match(vid):
+            return f"https://www.youtube.com/watch?v={vid}"
+
+    if host == "youtube.com" and parsed.path.startswith("/shorts/"):
+        vid = parsed.path[len("/shorts/") :].lstrip("/").split("/")[0]
+        if _VIDEO_ID_RE.match(vid):
+            return f"https://www.youtube.com/watch?v={vid}"
+
+    return url
+
+
+def _download_video_track(job: Job, url: str, job_dir: Path) -> None:
+    """Best-effort: download a video-only H.264/MP4 stream to video.mp4 for the
+    MP4 export (issue #219). The audio source is downloaded separately as
+    usual; this is a second, additive fetch so the audio pipeline is untouched.
+
+    Video-only MP4 needs no ffmpeg merge, so this can't break an audio-only job:
+    any failure (no progressive MP4 video, network error, unsupported codec) is
+    logged and swallowed, leaving has_video False. A cancel mid-download raises
+    JobCancelled, which the runner treats like any other cancellation.
+
+    Capped at VIDEO_MAX_HEIGHT to keep downloads reasonable -- a full song at
+    1080p is large, and the MP4 export doesn't need it."""
+
+    def vhook(d: dict) -> None:
+        if job.cancel_requested:
+            raise JobCancelled()
+        if d.get("status") == "downloading":
+            total = d.get("total_bytes") or d.get("total_bytes_estimate")
+            if total:
+                p = float(d.get("downloaded_bytes", 0)) / float(total)
+                _set(job, stage=f"Fetching video {int(p * 100)}%")
+
+    # Prefer H.264 (avc1) so the exported MP4 plays everywhere -- YouTube also
+    # serves AV1/VP9 in mp4 containers, which many players (Safari/iOS, older
+    # devices) can't decode. Fall back to any <=cap mp4 only if no avc1 exists.
+    max_height = get_video_max_height()
+    ydl_opts = {
+        "format": (
+            f"bestvideo[height<={max_height}][vcodec^=avc1]"
+            f"/bestvideo[height<={max_height}][ext=mp4]"
+        ),
+        "outtmpl": str(job_dir / "video.%(ext)s"),
+        "quiet": True,
+        "noprogress": True,
+        "noplaylist": True,
+        "allowed_extractors": _ALLOWED_EXTRACTORS,
+        "progress_hooks": [vhook],
+    }
+    # Point yt-dlp at the bundled ffmpeg in case a DASH stream needs remuxing;
+    # in portable builds ffmpeg is not on PATH.
+    if FFMPEG_DIR.is_dir():
+        ydl_opts["ffmpeg_location"] = str(FFMPEG_DIR)
+
+    _set(job, stage="Fetching video...")
+    try:
+        with YoutubeDL(ydl_opts) as ydl:
+            ydl.extract_info(url, download=True)
+    except JobCancelled:
+        raise
+    except Exception as exc:
+        if job.cancel_requested:
+            raise JobCancelled() from exc
+        logger.warning("[%s] video track unavailable (audio-only): %s", job.id, exc)
+
+    video = job_dir / "video.mp4"
+    if video.is_file() and video.stat().st_size > 0:
+        job.has_video = True
+    else:
+        # Drop any partial/non-mp4 leftover so the export endpoint sees nothing.
+        for f in job_dir.glob("video.*"):
+            f.unlink(missing_ok=True)
+
+
+def download(job: Job, url: str, job_dir: Path) -> Path:
+    url = normalize_youtube_url(url)
+    logger.info("[%s] download starting: %s", job.id, url)
+    _set(job, status="downloading", progress=0.0, stage="Processing...")
+
+    # Fetch metadata first (no download) so we can reject videos that are
+    # too long before wasting bandwidth and disk.
+    with YoutubeDL(
+        {"quiet": True, "noplaylist": True, "allowed_extractors": _ALLOWED_EXTRACTORS}
+    ) as ydl:
+        meta = ydl.extract_info(url, download=False) or {}
+    duration = meta.get("duration") or 0
+    max_duration = get_max_duration_sec()
+    if duration > max_duration:
+        mins = max_duration // 60
+        raise RuntimeError(f"Video is {int(duration // 60)} min -- limit is {mins} min")
+
+    def hook(d: dict) -> None:
+        # yt-dlp calls this on each chunk; raising here aborts the download.
+        # The runner unwraps yt-dlp's DownloadError and routes to JobCancelled.
+        if job.cancel_requested:
+            raise JobCancelled()
+        if d.get("status") == "downloading":
+            total = d.get("total_bytes") or d.get("total_bytes_estimate")
+            if total:
+                p = float(d.get("downloaded_bytes", 0)) / float(total)
+                _set(job, progress=p, stage=f"Downloading {int(p * 100)}%")
+        elif d.get("status") == "finished":
+            _set(job, progress=1.0, stage="Download complete")
+
+    # YouTube jobs additionally fetch the real video stream (below) for the
+    # MP4 export (issue #219). SoundCloud is audio-only and excluded.
+    is_youtube = url.startswith("https://www.youtube.com/")
+
+    # No postprocessors -- Demucs reads the raw audio container (webm/m4a/opus/...)
+    # directly via torchaudio + ffmpeg. Skipping the WAV transcode saves the slowest
+    # part of the download pipeline and a lot of disk.
+    ydl_opts = {
+        "format": "bestaudio/best",
+        "outtmpl": str(job_dir / "source.%(ext)s"),
+        "quiet": True,
+        "noprogress": True,
+        "noplaylist": True,
+        "allowed_extractors": _ALLOWED_EXTRACTORS,
+        "progress_hooks": [hook],
+    }
+    info: dict = {}
+    for attempt in range(_MAX_RETRIES + 1):
+        try:
+            with YoutubeDL(ydl_opts) as ydl:
+                info = ydl.extract_info(url, download=True) or {}
+            break
+        except Exception as exc:
+            if job.cancel_requested:
+                raise JobCancelled() from exc
+            if attempt < _MAX_RETRIES and _is_retriable(exc):
+                wait = _RETRY_BACKOFF[attempt]
+                logger.warning(
+                    "[%s] download attempt %d/%d failed (%s), retrying in %ds",
+                    job.id,
+                    attempt + 1,
+                    _MAX_RETRIES,
+                    exc,
+                    wait,
+                )
+                _set(job, stage=f"Network error — retrying ({attempt + 1}/{_MAX_RETRIES})...")
+                time.sleep(wait)
+            else:
+                raise
+
+    _set(
+        job,
+        title=info.get("title") or meta.get("title"),
+        duration_sec=info.get("duration") or duration,
+        thumbnail=info.get("thumbnail") or meta.get("thumbnail"),
+    )
+
+    raw_tags = [
+        t.strip().lower()
+        for t in (info.get("tags") or []) + (info.get("categories") or [])
+        if isinstance(t, str) and t.strip()
+    ]
+    seen: set[str] = set()
+    deduped = [t for t in raw_tags if not (t in seen or seen.add(t))]  # type: ignore[func-returns-value]
+    _set(job, tags=deduped[:8] or None)
+
+    # Best-effort: fetch the real video stream for the MP4 export.
+    # Non-fatal -- on any failure the job proceeds audio-only.
+    if is_youtube:
+        _download_video_track(job, url, job_dir)
+
+    candidates = sorted(job_dir.glob("source.*"))
+    if not candidates:
+        raise RuntimeError("yt-dlp finished but no source file was produced")
+    logger.info("[%s] download complete: %s", job.id, candidates[0].name)
+    return candidates[0]

app/pipeline/runner.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import shutil
+import subprocess
+from pathlib import Path
+
+from app.core.config import TIMEOUT_FFMPEG
+from app.core.models import Job, JobCancelled, _set
+from app.core.registry import persist as persist_registry
+from app.pipeline.analyze import analyze, compute_stem_presence
+from app.pipeline.collect import (
+    cleanup_source,
+    collect,
+    compute_stem_peaks,
+    make_original_track,
+    make_selected_mix,
+)
+from app.pipeline.download import download
+from app.pipeline.separate import separate
+
+logger = logging.getLogger("stemdeck.pipeline")
+
+
+def _rmtree(path: Path) -> None:
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError:
+        pass
+    except Exception:
+        logger.warning("failed to remove %s", path, exc_info=True)
+
+
+# Only one heavy job runs at a time -- Demucs is GPU/CPU-hungry.
+_pipeline_lock = asyncio.Semaphore(1)
+
+
+def _check_cancel(job: Job) -> None:
+    if job.cancel_requested:
+        raise JobCancelled()
+
+
+def _extract_video_track(job: Job, source: Path, job_dir: Path) -> None:
+    """For an .mp4 upload, preserve a silent video-only track at
+    video.mp4 so the studio can later mux it with a custom stem mix
+    into an MP4 (issue #219). Stream-copies the video (no
+    re-encode) -- fast and lossless.
+
+    Best-effort: an .mp4 with no video stream (audio-only container)
+    fails harmlessly and leaves has_video false."""
+    from app.core.config import ffmpeg_executable
+
+    dest = job_dir / "video.mp4"
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(source),
+        "-an",  # drop audio -- the mix is added at export time
+        "-c:v",
+        "copy",
+        "-movflags",
+        "+faststart",
+        "-y",
+        str(dest),
+    ]
+    result = subprocess.run(cmd, capture_output=True, timeout=TIMEOUT_FFMPEG)
+    if result.returncode != 0 or not dest.is_file() or dest.stat().st_size == 0:
+        dest.unlink(missing_ok=True)
+        logger.info("no video track preserved for job %s (source has no video stream?)", job.id)
+        return
+    job.has_video = True
+
+
+def _prepare_local_source(job: Job, source: Path, job_dir: Path) -> Path:
+    """Transcode any local upload to 16-bit 44.1 kHz stereo WAV before
+    handing it to Demucs. Normalises MP3 and non-standard WAV formats
+    (24-bit, 32-bit float, high sample rate, multi-channel) that Demucs
+    would otherwise process silently and output as silence.
+
+    For .mp4 uploads, first preserves a silent video.mp4 for later
+    MP4 export. Deletes the original source file after a
+    successful transcode."""
+    from app.core.config import ffmpeg_executable
+
+    dest = job_dir / "source.wav"
+    if source.resolve() == dest.resolve():
+        return source
+
+    _set(job, stage="Preparing audio...")
+    if source.suffix.lower() == ".mp4":
+        _extract_video_track(job, source, job_dir)
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(source),
+        "-ar",
+        "44100",
+        "-ac",
+        "2",
+        "-sample_fmt",
+        "s16",
+        "-y",
+        str(dest),
+    ]
+    result = subprocess.run(cmd, capture_output=True, timeout=TIMEOUT_FFMPEG)
+    if result.returncode != 0:
+        raise RuntimeError(
+            "ffmpeg transcode failed: " + result.stderr.decode("utf-8", errors="replace").strip()
+        )
+    source.unlink(missing_ok=True)
+    return dest
+
+
+def _run_common(job: Job, source: Path, job_dir: Path) -> None:
+    """Analyze → separate → collect → mix. Shared by both YouTube and local
+    upload pipelines after their respective source acquisition steps."""
+    _check_cancel(job)
+    analyze(job, source)
+    _check_cancel(job)
+    stems_root = separate(job, source, job_dir)
+    found = collect(job, stems_root, job_dir)
+    stems_dir = job_dir / "stems"
+    job.stem_presence = compute_stem_presence(stems_dir, found)
+    # Source (100-300 MB or the local upload) is no longer needed after
+    # collect; delete it before the ffmpeg amix steps in case scratch space
+    # is tight.
+    cleanup_source(job_dir)
+    job.stems = [{"name": name, "url": f"/api/jobs/{job.id}/stems/{name}.wav"} for name in found]
+    _check_cancel(job)
+    _set(job, stage="Mixing tracks...")
+    original_path = make_original_track(job, job_dir, stems_dir)
+    if original_path is not None:
+        job.stems.insert(
+            0,
+            {
+                "name": "original",
+                "url": f"/api/jobs/{job.id}/stems/original.wav",
+            },
+        )
+    _check_cancel(job)
+    mix_path = make_selected_mix(job, stems_dir, found)
+    if mix_path is not None:
+        job.mix_url = f"/api/jobs/{job.id}/stems/{mix_path.name}"
+    _check_cancel(job)
+
+    all_stem_names = [s["name"] for s in job.stems]
+    if mix_path is not None and mix_path.stem not in all_stem_names:
+        all_stem_names.append(mix_path.stem)
+    compute_stem_peaks(stems_dir, all_stem_names)
+
+
+def _run_blocking(job: Job, url: str, job_dir: Path) -> None:
+    _check_cancel(job)
+    source = download(job, url, job_dir)
+    _run_common(job, source, job_dir)
+
+
+def _run_local_blocking(job: Job, source_path: Path, job_dir: Path) -> None:
+    _check_cancel(job)
+    source = _prepare_local_source(job, source_path, job_dir)
+    _run_common(job, source, job_dir)
+
+
+def _write_metadata(job: Job, job_dir: Path) -> None:
+    meta = {
+        "title": job.title,
+        "thumbnail": job.thumbnail,
+        "duration_sec": job.duration_sec,
+        "bpm": job.bpm,
+        "key": job.key,
+        "scale": job.scale,
+        "key_confidence": job.key_confidence,
+        "lufs": job.lufs,
+        "peak_db": job.peak_db,
+        "dynamic_range": job.dynamic_range,
+        "tempo_stability": job.tempo_stability,
+        "stem_presence": job.stem_presence,
+        "tags": job.tags,
+        "has_video": job.has_video,
+    }
+    try:
+        (job_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8")
+    except OSError:
+        logger.warning("could not write metadata.json for job %s", job.id, exc_info=True)
+
+
+async def _run_async(
+    job: Job,
+    job_dir: Path,
+    jobs_dir: Path,
+    blocking_fn,
+    *fn_args: object,
+    error_msg: str = "Audio processing failed. Please try again.",
+) -> None:
+    """Common async wrapper: acquires the pipeline lock, runs blocking_fn in a
+    thread, then handles success / cancel / error outcomes uniformly."""
+    try:
+        async with _pipeline_lock:
+            await asyncio.to_thread(blocking_fn, job, *fn_args, job_dir)
+    except Exception as e:
+        if not isinstance(e, JobCancelled) and not job.cancel_requested:
+            logger.exception("pipeline failed for job %s: %s", job.id, e)
+            _set(job, status="error", stage="Error: Processing failed", error=error_msg)
+            persist_registry(jobs_dir)
+            _rmtree(job_dir)
+            return
+        logger.info(
+            "pipeline cancelled%s for job %s",
+            " (wrapped)" if not isinstance(e, JobCancelled) else "",
+            job.id,
+        )
+        _set(job, status="cancelled", stage="Cancelled")
+        persist_registry(jobs_dir)
+        _rmtree(job_dir)
+        return
+    _set(job, status="done", progress=1.0, stage="Done")
+    _write_metadata(job, job_dir)
+    persist_registry(jobs_dir)
+
+
+async def run_pipeline(job: Job, url: str, jobs_dir: Path) -> None:
+    job_dir = jobs_dir / job.id
+    try:
+        job_dir.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        logger.exception("pipeline failed for job %s: %s", job.id, e)
+        _set(
+            job,
+            status="error",
+            stage="Error: Processing failed",
+            error="Audio processing failed. Please try another video.",
+        )
+        persist_registry(jobs_dir)
+        return
+    await _run_async(
+        job,
+        job_dir,
+        jobs_dir,
+        _run_blocking,
+        url,
+        error_msg="Audio processing failed. Please try another video.",
+    )
+
+
+async def run_local_pipeline(job: Job, source_path: Path, jobs_dir: Path) -> None:
+    """Run the stem-separation pipeline for a locally uploaded file.
+    The job directory and source file are already present on disk (created
+    by the API handler before this task is scheduled)."""
+    job_dir = jobs_dir / job.id
+    await _run_async(job, job_dir, jobs_dir, _run_local_blocking, source_path)

app/pipeline/separate.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import logging
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+from app.core.config import DEMUCS_DEVICE, DEMUCS_MODEL, TIMEOUT_DEMUCS_STALL
+from app.core.models import Job, JobCancelled, _set
+from app.core.registry import set_proc
+
+logger = logging.getLogger("stemdeck.pipeline")
+
+_PCT_RE = re.compile(r"(\d{1,3})%")
+# Terminate demucs if stderr produces no output for this many seconds.
+# GPU processing can be silent for minutes; 30 min covers legitimate pauses
+# while still catching genuine hangs (GPU deadlock, OOM stall, etc.).
+
+
+def separate(job: Job, source: Path, job_dir: Path) -> Path:
+    _set(job, status="separating", progress=0.0, stage="Separating stems...")
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "demucs",
+        "-n",
+        DEMUCS_MODEL,
+        "-d",
+        DEMUCS_DEVICE,
+        "-o",
+        str(job_dir),
+        str(source),
+    ]
+    env = os.environ.copy()
+    try:
+        import certifi
+
+        env.setdefault("SSL_CERT_FILE", certifi.where())
+        env.setdefault("REQUESTS_CA_BUNDLE", certifi.where())
+    except ModuleNotFoundError:
+        pass
+
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=0,
+        env=env,
+    )
+    if proc.stderr is None:
+        raise RuntimeError("demucs subprocess has no stderr pipe")
+    set_proc(job.id, proc)
+
+    # tqdm uses \r to redraw -- read char-by-char and split on \r or \n.
+    # Keep the last few non-progress lines so we can surface them if demucs
+    # exits non-zero (otherwise the only signal would be a bare exit code).
+    buf = ""
+    tail: list[str] = []
+    last_output: list[float] = [time.monotonic()]
+    # Event set by the reader loop when the process exits normally so the
+    # watchdog can wake up immediately instead of waiting out its 30 s sleep.
+    _done_evt = threading.Event()
+
+    def _watchdog() -> None:
+        while not _done_evt.wait(timeout=30):
+            if proc.poll() is not None:
+                return
+            if time.monotonic() - last_output[0] > TIMEOUT_DEMUCS_STALL:
+                logger.warning(
+                    "demucs stalled for %ss with no output, terminating job %s",
+                    TIMEOUT_DEMUCS_STALL,
+                    job.id,
+                )
+                proc.terminate()
+                return
+
+    wt = threading.Thread(target=_watchdog, daemon=True)
+    wt.start()
+    try:
+        while True:
+            ch = proc.stderr.read(1)
+            if not ch:
+                break
+            last_output[0] = time.monotonic()
+            if ch in ("\r", "\n"):
+                line = buf.strip()
+                buf = ""
+                if not line:
+                    continue
+                m = _PCT_RE.search(line)
+                if m:
+                    pct = max(0, min(100, int(m.group(1))))
+                    _set(job, progress=pct / 100.0, stage=f"Separating {pct}%")
+                else:
+                    tail.append(line)
+                    if len(tail) > 40:
+                        tail.pop(0)
+            else:
+                buf += ch
+
+        proc.wait()
+    finally:
+        _done_evt.set()
+        set_proc(job.id, None)
+        wt.join(timeout=2)
+
+    # POST /cancel calls proc.terminate() directly, which causes the read loop
+    # above to hit EOF and proc.wait() to return a nonzero status. Translate
+    # that into JobCancelled before the generic "demucs failed" path.
+    if job.cancel_requested:
+        raise JobCancelled()
+    if proc.returncode != 0:
+        detail = "\n".join(tail[-15:]) if tail else "(no stderr captured)"
+        logger.error("[%s] demucs exited %s; tail:\n%s", job.id, proc.returncode, detail)
+        last = tail[-1] if tail else f"exit status {proc.returncode}"
+        raise RuntimeError(f"demucs failed: {last}")
+
+    stems_root = job_dir / DEMUCS_MODEL / source.stem
+    if not stems_root.is_dir():
+        raise RuntimeError(f"demucs output not found at {stems_root}")
+    return stems_root