chore: import upstream snapshot with attribution

app/pipeline/__init__.py

@@ -0,0 +1,3 @@
+from app.pipeline.runner import run_local_pipeline, run_pipeline
+
+__all__ = ["run_pipeline", "run_local_pipeline"]

app/pipeline/analyze.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import logging
+import subprocess
+from pathlib import Path
+
+from app.core.config import JOBS_DIR, TIMEOUT_ANALYZE, ffmpeg_executable
+from app.core.models import Job, _set
+
+logger = logging.getLogger("stemdeck.analyze")
+
+# Albrecht-Shanahan key profiles, derived from a corpus of popular music
+# (Albrecht & Shanahan, 2013). Critically, the minor profile here weights
+# b7 high (3.48) and M7 low (0.81) — the opposite of Temperley/Kostka-Payne,
+# which were derived from Bach chorales and bias toward harmonic minor's
+# leading tone. Pop/rock uses natural minor: the b7 is the diatonic
+# seventh and rings out constantly (e.g. open D in "Come As You Are",
+# which is in E minor and uses D as the b7). Values rescaled so that the
+# tonic weight is ≈5 to match the prior code's magnitude.
+_MAJOR_PROFILE = (
+    5.47,
+    0.14,
+    2.55,
+    0.14,
+    3.15,
+    2.16,
+    0.37,
+    4.92,
+    0.21,
+    1.84,
+    0.18,
+    1.86,
+)
+_MINOR_PROFILE = (
+    5.06,
+    0.14,
+    2.42,
+    2.42,
+    0.35,
+    1.96,
+    0.35,
+    4.16,
+    2.53,
+    0.28,
+    2.67,
+    0.62,
+)
+_PITCHES = ("C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B")
+
+# When the best-major and best-minor scores are this close, we prefer
+# minor. Pop/rock has a strong minor-mode prior; the algorithm often
+# walks toward the relative major because of an ostinato bass note
+# (e.g. "Come As You Are" hammers the open D string in an E minor song),
+# and minor is the better default when the call is genuinely ambiguous.
+_MINOR_TIE_BREAK_FRAC = 0.05
+
+
+def _correlate(profile: tuple[float, ...], chroma: list[float], shift: int) -> float:
+    n = len(profile)
+    rotated = [chroma[(i + shift) % n] for i in range(n)]
+    mean_p = sum(profile) / n
+    mean_c = sum(rotated) / n
+    num = sum((profile[i] - mean_p) * (rotated[i] - mean_c) for i in range(n))
+    denom_p = sum((profile[i] - mean_p) ** 2 for i in range(n)) ** 0.5
+    denom_c = sum((rotated[i] - mean_c) ** 2 for i in range(n)) ** 0.5
+    if denom_p == 0 or denom_c == 0:
+        return 0.0
+    return num / (denom_p * denom_c)
+
+
+def _detect_key(chroma_mean: list[float]) -> tuple[str, str, int]:
+    """Find the best-matching key by combining profile correlation with
+    root prominence. The Pearson correlation alone is fooled by relative
+    keys whose diatonic notes happen to overlap with the song's loud
+    pitches but whose own tonic is weak (e.g. picking A minor for an
+    E-minor song because E is its 5th and D is its 4th). Weighting by
+    the candidate root's chroma value forces the algorithm to also
+    confirm 'is this proposed tonic actually loud in the audio?'.
+    Logs the chroma vector and top-5 candidates for diagnostics.
+
+    Returns (label, scale_name, confidence_pct).
+    - label:        e.g. "G# maj"
+    - scale_name:   "Major" or "Natural Minor"
+    - confidence_pct: 0-100, derived from the gap between the winning
+                    candidate and the runner-up, normalized so a clear
+                    win ranks high and a near-tie ranks low."""
+    raw: list[tuple[float, float, str, int]] = []  # (weighted, pearson, label, root_idx)
+    for shift in range(12):
+        root_strength = chroma_mean[shift]
+        pearson_maj = _correlate(_MAJOR_PROFILE, chroma_mean, shift)
+        pearson_min = _correlate(_MINOR_PROFILE, chroma_mean, shift)
+        # Multiplicative root weighting. Pearson can be negative; when
+        # it is, a low-chroma root makes things less negative (closer to
+        # zero), which is actually the desired ordering.
+        raw.append((pearson_maj * root_strength, pearson_maj, f"{_PITCHES[shift]} maj", shift))
+        raw.append((pearson_min * root_strength, pearson_min, f"{_PITCHES[shift]} min", shift))
+    raw.sort(key=lambda x: x[0], reverse=True)
+
+    # Diagnostic log: chroma profile + top 5 candidates with both raw
+    # and weighted scores. Lets us see what the algorithm is "hearing".
+    chroma_str = ", ".join(f"{_PITCHES[i]}={chroma_mean[i]:.3f}" for i in range(12))
+    top5_str = ", ".join(
+        f"{label}={weighted:+.3f}(p{pearson:+.2f}*r{chroma_mean[idx]:.2f})"
+        for weighted, pearson, label, idx in raw[:5]
+    )
+    logger.debug("chroma: %s", chroma_str)
+    logger.debug("key candidates (top 5): %s", top5_str)
+
+    # Pick best major and best minor for the tie-break, both by the
+    # weighted score.
+    best_maj = next(c for c in raw if c[2].endswith("maj"))
+    best_min = next(c for c in raw if c[2].endswith("min"))
+
+    gap = abs(best_maj[0] - best_min[0])
+    threshold = max(abs(best_maj[0]), abs(best_min[0])) * _MINOR_TIE_BREAK_FRAC
+    # Near-tie -> prefer minor (pop/rock prior); clear winner -> use it.
+    winner = (best_maj if best_maj[0] > best_min[0] else best_min) if gap > threshold else best_min
+
+    # Confidence: gap between the winner and the runner-up that *isn't*
+    # the relative major/minor of the winner (those will always be near-
+    # ties with the algorithm's profile-correlation approach, so they
+    # tell us nothing about real ambiguity). Normalize so a healthy 0.15
+    # gap = 100% confident; tiny gap = 0%.
+    runner_up = next(c for c in raw if c[2] != winner[2])
+    confidence_score = winner[0] - runner_up[0]
+    confidence_pct = max(0, min(100, round(confidence_score / 0.15 * 100)))
+
+    label = winner[2]
+    scale_name = "Major" if label.endswith("maj") else "Natural Minor"
+    return label, scale_name, confidence_pct
+
+
+def _measure_loudness(y: object, sr: int) -> tuple[float | None, float | None]:
+    """Compute integrated loudness (LUFS, BS.1770) and sample peak (dBFS)
+    of the loaded mono signal. Returns (lufs, peak_db); either may be
+    None on failure or silence. We use sample peak rather than oversampled
+    true peak -- the difference is typically <1 dB and not worth the 4x
+    resample cost for a display field."""
+    import numpy as np
+
+    if y is None or getattr(y, "size", 0) == 0:
+        return None, None
+
+    peak_lin = float(np.abs(y).max())
+    peak_db = 20.0 * float(np.log10(peak_lin)) if peak_lin > 1e-9 else None
+
+    lufs: float | None = None
+    try:
+        import pyloudnorm as pyln
+
+        meter = pyln.Meter(sr)  # BS.1770-4 with default 400ms blocks
+        lufs_raw = float(meter.integrated_loudness(y))
+        # pyloudnorm returns -inf for silence; surface as None instead so
+        # the frontend can hide the field rather than render "-inf LUFS".
+        if np.isfinite(lufs_raw):
+            lufs = lufs_raw
+    except (ImportError, ValueError) as e:
+        # ValueError fires if the clip is shorter than the gating window.
+        logger.warning("LUFS measurement failed: %s", e)
+    return lufs, peak_db
+
+
+def _load_audio_ffmpeg(
+    source: Path, sr: int = 22050, duration: float = 180.0
+) -> tuple[object, int] | None:
+    """Decode `source` to a mono float32 numpy array at `sr` via ffmpeg.
+    Bypasses librosa's deprecated audioread fallback (which fires a
+    FutureWarning on .webm/.m4a/.opus inputs because soundfile can't
+    read those directly). Returns (samples, sr) or None on failure."""
+    import numpy as np
+
+    # Defence in depth: even though `source` is constructed by the server
+    # (never user-typed), confirm it's a real file inside JOBS_DIR before
+    # handing it to a subprocess. Belt-and-suspenders against a future
+    # caller change that would let a path slip in from elsewhere.
+    resolved = source.resolve()
+    jobs_resolved = JOBS_DIR.resolve()
+    if not resolved.is_file():
+        logger.warning("analyze source is not a file: %s", source)
+        return None
+    if not resolved.is_relative_to(jobs_resolved):
+        logger.warning(
+            "analyze source escapes JOBS_DIR (%s not under %s)",
+            resolved,
+            jobs_resolved,
+        )
+        return None
+
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(resolved),
+        "-ac",
+        "1",  # mono
+        "-ar",
+        str(sr),  # resample
+        "-f",
+        "f32le",  # raw 32-bit float little-endian
+        "-t",
+        str(duration),  # cap input duration
+        "-",  # write to stdout
+    ]
+    try:
+        proc = subprocess.run(cmd, capture_output=True, check=True, timeout=TIMEOUT_ANALYZE)
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as e:
+        logger.warning("ffmpeg decode failed for %s: %s", source, e)
+        return None
+    y = np.frombuffer(proc.stdout, dtype=np.float32)
+    if y.size == 0:
+        return None
+    return y, sr
+
+
+def compute_stem_presence(stems_dir: Path, selected_stems: list[str]) -> dict[str, int]:
+    """Load each extracted stem WAV, compute mean absolute amplitude, normalize
+    to 0-100. Only the stems that were selected (and therefore extracted) are
+    measured; the rest are omitted from the returned dict."""
+    import numpy as np
+
+    result: dict[str, int] = {}
+    rms_values: dict[str, float] = {}
+
+    for name in selected_stems:
+        wav_path = stems_dir / f"{name}.wav"
+        if not wav_path.is_file():
+            continue
+        loaded = _load_audio_ffmpeg(wav_path, sr=22050, duration=180.0)
+        if loaded is None:
+            continue
+        y, _ = loaded
+        rms_values[name] = float(np.sqrt(np.mean(y**2)))
+
+    if not rms_values:
+        return result
+
+    max_rms = max(rms_values.values())
+    if max_rms < 1e-9:
+        return {name: 0 for name in rms_values}
+
+    for name, rms in rms_values.items():
+        result[name] = max(0, min(100, round(rms / max_rms * 100)))
+
+    return result
+
+
+def analyze(job: Job, source: Path) -> tuple[int | None, str | None]:
+    """Best-effort BPM and key detection. On failure, returns (None, None)
+    and leaves job fields untouched -- the chips stay as placeholders."""
+    logger.info("analyze: entering for job %s, source=%s", job.id, source)
+    _set(job, status="analyzing", progress=0.0, stage="Analyzing audio...")
+    try:
+        import librosa
+    except ImportError:
+        logger.warning("librosa not installed -- skipping BPM/key analysis")
+        return None, None
+
+    try:
+        # Analyse the first 180 s. Decode via ffmpeg directly into numpy
+        # to avoid librosa's deprecated audioread fallback for
+        # .webm/.m4a/.opus inputs.
+        loaded = _load_audio_ffmpeg(source, sr=22050, duration=180.0)
+        if loaded is None:
+            return None, None
+        y, sr = loaded
+
+        # Harmonic / percussive separation. Beat tracking sees a cleaner
+        # onset envelope on the percussive component; chroma sees a
+        # cleaner pitch profile on the harmonic component (no cymbal
+        # smear, no kick fundamentals leaking in).
+        y_harmonic, y_percussive = librosa.effects.hpss(y)
+
+        tempo_arr, beat_frames = librosa.beat.beat_track(y=y_percussive, sr=sr)
+        try:
+            tempo = float(tempo_arr[0])  # type: ignore[index]
+        except (TypeError, IndexError):
+            tempo = float(tempo_arr)
+        bpm = int(round(tempo)) if tempo > 0 else None
+
+        # chroma_cqt is constant-Q based — better pitch resolution than
+        # chroma_stft, especially in the bass register where the open
+        # strings of a guitar live.
+        chroma = librosa.feature.chroma_cqt(y=y_harmonic, sr=sr)
+        chroma_mean = chroma.mean(axis=1).tolist()
+        if any(chroma_mean):
+            key, scale, key_confidence = _detect_key(chroma_mean)
+        else:
+            key, scale, key_confidence = None, None, None
+
+        # LUFS / peak. Computed on the same 22 kHz mono buffer; this
+        # loses a few dB of accuracy vs full-sample-rate stereo, but
+        # it's good enough for a UI display and adds ~50 ms to analyze.
+        lufs, peak_db = _measure_loudness(y, sr)
+
+        dynamic_range: float | None = None
+        if lufs is not None and peak_db is not None:
+            dynamic_range = round(peak_db - lufs, 1)
+
+        # Beat interval coefficient of variation → stability 0-100.
+        # CV = std/mean of inter-beat intervals; CV=0 is perfectly metronomic.
+        tempo_stability: int | None = None
+        import numpy as np
+
+        beat_times = librosa.frames_to_time(beat_frames, sr=sr)
+        if len(beat_times) > 2:
+            intervals = np.diff(beat_times)
+            mean_iv = float(intervals.mean())
+            if mean_iv > 0:
+                cv = float(intervals.std() / mean_iv)
+                tempo_stability = max(0, min(100, round((1 - min(cv, 1)) * 100)))
+
+        _set(
+            job,
+            bpm=bpm,
+            key=key,
+            scale=scale,
+            key_confidence=key_confidence,
+            lufs=lufs,
+            peak_db=peak_db,
+            dynamic_range=dynamic_range,
+            tempo_stability=tempo_stability,
+            progress=1.0,
+            stage="Analysis complete",
+        )
+        return bpm, key
+    except Exception as e:
+        logger.exception("analyze failed for job %s", job.id)
+        _set(job, stage=f"Analysis skipped ({e})")
+        return None, None

app/pipeline/collect.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+from app.core.config import (
+    DEMUCS_MODEL,
+    JOB_TTL_SECONDS,
+    STEM_NAMES,
+    TIMEOUT_FFMPEG,
+    ffmpeg_executable,
+)
+from app.core.models import Job
+from app.core.registry import all_jobs as registry_all
+from app.core.registry import persist as registry_persist
+from app.core.registry import remove as registry_remove
+from app.core.registry import set_proc
+
+logger = logging.getLogger("stemdeck.collect")
+
+
+def _rmtree(path: Path) -> None:
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError:
+        pass
+    except Exception:
+        logger.warning("failed to remove %s", path, exc_info=True)
+
+
+def _run_ffmpeg(job: Job, cmd: list[str]) -> bool:
+    """Run an ffmpeg command, registering the subprocess with the job
+    registry so POST /api/jobs/{id}/cancel can terminate it. Returns
+    True on success, False on failure or external termination.
+
+    Without registering the proc, an in-flight ffmpeg amix would block
+    cancellation for up to its 300s timeout -- the cancel flag is set
+    but the runner can't see it until subprocess.run returns. With
+    set_proc, the cancel API can call proc.terminate() directly and
+    communicate() returns within ~1s with a non-zero returncode."""
+    proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+    set_proc(job.id, proc)
+    try:
+        try:
+            _, stderr = proc.communicate(timeout=TIMEOUT_FFMPEG)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.communicate()
+            logger.warning("ffmpeg timed out for job %s", job.id)
+            return False
+        if proc.returncode != 0:
+            tail = (stderr or b"").decode(errors="replace").splitlines()[-3:]
+            logger.warning(
+                "ffmpeg exit %s for job %s: %s",
+                proc.returncode,
+                job.id,
+                " | ".join(tail) or "(no stderr)",
+            )
+            return False
+        return True
+    finally:
+        set_proc(job.id, None)
+
+
+_TERMINAL = frozenset(("done", "error", "cancelled"))
+
+
+def collect(job: Job, stems_root: Path, job_dir: Path) -> list[str]:
+    """Move Demucs-emitted stems into the job's stems/ dir and clean up
+    the demucs intermediate dir. Does NOT delete the source download --
+    cleanup_source() is called by the runner after any post-processing
+    that needs to re-encode the source (e.g. building original.wav)."""
+    target_dir = job_dir / "stems"
+    target_dir.mkdir(exist_ok=True)
+    found: list[str] = []
+    for name in STEM_NAMES:
+        src = stems_root / f"{name}.wav"
+        if src.exists():
+            shutil.move(str(src), target_dir / f"{name}.wav")
+            found.append(name)
+    _rmtree(job_dir / DEMUCS_MODEL)
+    if not found:
+        raise RuntimeError("no stems produced by demucs")
+    return found
+
+
+def cleanup_source(job_dir: Path) -> None:
+    """Delete the source audio file. Called after collect AND after any
+    post-processing that re-encodes the source (make_original_track).
+    The source is 100-300 MB, so getting rid of it is the bulk of disk
+    reclaim per job; only the stems remain."""
+    for f in job_dir.glob("source.*"):
+        f.unlink(missing_ok=True)
+
+
+def make_original_track(job: Job, job_dir: Path, stems_dir: Path) -> Path | None:
+    """Build the "Original" backing track at stems/original.wav as the
+    sum of the stems the user did NOT select. This way the studio can
+    play (original + each selected stem) and reconstruct the full song
+    without doubling the selected stems -- which is what would happen
+    if "original" were the raw source download (drum hits in original
+    + isolated drums.wav = drums at 2x amplitude).
+
+    Skipped when the user kept all 6 stems (no complement to mix) or
+    when none of the unselected stem WAVs are on disk."""
+    unselected = [s for s in STEM_NAMES if s not in job.selected_stems]
+    inputs = [stems_dir / f"{name}.wav" for name in unselected]
+    inputs = [p for p in inputs if p.exists()]
+    if not inputs:
+        return None
+    out = stems_dir / "original.wav"
+    cmd: list[str] = [
+        ffmpeg_executable(),
+        "-y",
+        "-nostdin",
+        "-loglevel",
+        "error",
+    ]
+    for p in inputs:
+        cmd += ["-i", str(p)]
+    if len(inputs) == 1:
+        # Single complement stem -- copy as-is so we still produce a
+        # canonical mix.wav-shaped output without invoking amix on a
+        # 1-input graph (which is a no-op anyway).
+        cmd += ["-c:a", "pcm_s16le", str(out)]
+    else:
+        filter_inputs = "".join(f"[{i}:a]" for i in range(len(inputs)))
+        cmd += [
+            "-filter_complex",
+            f"{filter_inputs}amix=inputs={len(inputs)}:normalize=0",
+            "-c:a",
+            "pcm_s16le",
+            str(out),
+        ]
+    return out if _run_ffmpeg(job, cmd) else None
+
+
+def make_selected_mix(job: Job, stems_dir: Path, found: list[str]) -> Path | None:
+    """If the user picked a strict subset of stems at submit time,
+    sum those stems with ffmpeg amix into mix.wav. Returns the output
+    path on success, or None when there's nothing to mix.
+
+    Returns the existing single stem path (no ffmpeg) if exactly one
+    stem was selected -- copying it to mix.wav would be 30 MB of
+    duplicate data. The caller uses the returned path's name for the
+    download URL, so a single-stem selection points the Download Mix
+    button directly at the existing stem file.
+
+    amix normalize=0 keeps stem amplitudes as-is. Demucs separations
+    sum back to (close to) the original signal, so a 2-stem subset
+    fits comfortably below 0 dBFS without normalization headroom."""
+    selected = [s for s in job.selected_stems if s in found]
+    if not selected:
+        return None
+    if len(selected) == 1:
+        return stems_dir / f"{selected[0]}.wav"
+    inputs = [stems_dir / f"{name}.wav" for name in selected]
+    out = stems_dir / "mix.wav"
+    cmd: list[str] = [
+        ffmpeg_executable(),
+        "-y",
+        "-nostdin",
+        "-loglevel",
+        "error",
+    ]
+    for p in inputs:
+        cmd += ["-i", str(p)]
+    filter_inputs = "".join(f"[{i}:a]" for i in range(len(inputs)))
+    cmd += [
+        "-filter_complex",
+        f"{filter_inputs}amix=inputs={len(inputs)}:normalize=0",
+        "-c:a",
+        "pcm_s16le",
+        str(out),
+    ]
+    return out if _run_ffmpeg(job, cmd) else None
+
+
+_PEAK_POINTS = 1500  # matches OVERVIEW_WAVE_POINTS in player.js
+
+
+def compute_stem_peaks(stems_dir: Path, stem_names: list[str]) -> None:
+    """Compute and cache [min, max] waveform peaks for each stem.
+    Failure is non-fatal — missing peaks.json degrades to client-side decode."""
+    peaks: dict[str, list[list[float]]] = {}
+    for name in stem_names:
+        path = stems_dir / f"{name}.wav"
+        if not path.is_file():
+            continue
+        try:
+            data, _ = sf.read(path, dtype="float32", always_2d=True)
+            ch = data[:, 0]
+            n = len(ch)
+            if n == 0:
+                continue
+            chunk = max(1, n // _PEAK_POINTS)
+            result: list[list[float]] = []
+            for i in range(0, n, chunk):
+                block = ch[i : i + chunk]
+                result.append([float(np.min(block)), float(np.max(block))])
+            peaks[name] = result[:_PEAK_POINTS]
+        except Exception:
+            logger.warning("could not compute peaks for %s/%s", stems_dir.name, name, exc_info=True)
+
+    if not peaks:
+        return
+
+    try:
+        tmp = stems_dir / "peaks.json.tmp"
+        tmp.write_text(json.dumps(peaks), encoding="utf-8")
+        tmp.replace(stems_dir / "peaks.json")
+    except Exception:
+        logger.warning("could not write peaks.json for %s", stems_dir.name, exc_info=True)
+
+
+def sweep_old_jobs(jobs_dir: Path) -> None:
+    """Delete job directories older than JOB_TTL_SECONDS and remove them from
+    the in-memory registry. Called hourly from the background sweep loop
+    started at app startup.
+
+    Prefers Job.created_at over directory mtime (which can be touched by
+    unrelated filesystem events), and never deletes the directory of an
+    active (non-terminal) registered job even if its timestamp looks old.
+    Falls back to mtime for orphan directories left over from a previous
+    server run, since the registry is in-memory only."""
+    cutoff = time.time() - JOB_TTL_SECONDS
+    if not jobs_dir.is_dir():
+        return
+    jobs = registry_all()
+    removed = False
+    for d in jobs_dir.iterdir():
+        if not d.is_dir():
+            continue
+        job = jobs.get(d.name)
+        if job is not None:
+            if job.status not in _TERMINAL:
+                continue  # never delete an active job's working dir
+            if job.created_at >= cutoff:
+                continue
+        elif d.stat().st_mtime >= cutoff:
+            continue
+        _rmtree(d)
+        registry_remove(d.name)
+        removed = True
+    if removed:
+        registry_persist(jobs_dir)

app/pipeline/download.py

@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import logging
+import re
+import time
+import urllib.parse
+from pathlib import Path
+
+from yt_dlp import YoutubeDL
+
+from app.core.config import FFMPEG_DIR
+from app.core.models import Job, JobCancelled, _set
+from app.core.settings import get_max_duration_sec, get_video_max_height
+
+logger = logging.getLogger("stemdeck.download")
+
+_MAX_RETRIES = 3
+_RETRY_BACKOFF = (2, 4, 8)  # seconds between attempts
+
+# Errors worth retrying — transient network blips.
+_RETRIABLE = (
+    "connection reset",
+    "ssl",
+    "timed out",
+    "network is unreachable",
+    "temporary failure",
+    "unable to download",
+    "read timed out",
+    "remotedisconnected",
+    "broken pipe",
+    "connection refused",
+)
+
+# Errors that will never succeed on retry — reject immediately.
+_NON_RETRIABLE = (
+    "private video",
+    "video unavailable",
+    "has been removed",
+    "http error 404",
+    "http error 403",
+    "not available in your country",
+    "age-restricted",
+)
+
+
+def _is_retriable(exc: Exception) -> bool:
+    msg = str(exc).lower()
+    if any(s in msg for s in _NON_RETRIABLE):
+        return False
+    return any(s in msg for s in _RETRIABLE)
+
+
+_VIDEO_ID_RE = re.compile(r"^[A-Za-z0-9_-]{11}$")
+_YOUTUBE_HOSTS = frozenset(
+    (
+        "youtube.com",
+        "www.youtube.com",
+        "m.youtube.com",
+        "music.youtube.com",
+        "youtu.be",
+    )
+)
+# Note: on.soundcloud.com (the share shortener) is intentionally excluded — it
+# redirects to arbitrary targets, which is an SSRF vector once handed to yt-dlp
+# (#173). Users must paste the full soundcloud.com URL.
+_SOUNDCLOUD_HOSTS = frozenset(("soundcloud.com", "www.soundcloud.com"))
+_ALLOWED_HOSTS = _YOUTUBE_HOSTS | _SOUNDCLOUD_HOSTS
+
+# Restrict yt-dlp to the extractors we actually support. Crucially this excludes
+# the "generic" extractor, so even a URL that slips past host validation cannot
+# make yt-dlp fetch an arbitrary host/redirect target (#173).
+_ALLOWED_EXTRACTORS = ["youtube", "soundcloud"]
+
+
+class InvalidYouTubeURL(ValueError):
+    """Raised at the API boundary for URLs we won't hand to yt-dlp."""
+
+
+def validate_youtube_url(url: str) -> str:
+    """Reject anything that isn't an http(s) URL on a known supported host.
+    YouTube URLs are normalized to single-video form; SoundCloud URLs are
+    passed through as-is. Gives callers a clean 422 instead of a yt-dlp
+    extractor stack trace."""
+    if not isinstance(url, str) or not url.strip():
+        raise InvalidYouTubeURL("URL is required")
+    url = url.strip()
+    try:
+        parsed = urllib.parse.urlparse(url)
+    except Exception as e:
+        raise InvalidYouTubeURL(f"could not parse URL: {e}") from e
+    if parsed.scheme not in ("http", "https"):
+        raise InvalidYouTubeURL("URL must use http or https")
+    host = (parsed.hostname or "").lower()
+    if host not in _ALLOWED_HOSTS:
+        raise InvalidYouTubeURL(f"unsupported host: {host or '(empty)'}")
+
+    if host in _SOUNDCLOUD_HOSTS:
+        return url
+
+    normalized = normalize_youtube_url(url)
+    # normalize_youtube_url returns the original on playlist-only URLs with
+    # no derivable seed video. We always expect the canonical watch?v=... form.
+    if not normalized.startswith("https://www.youtube.com/watch?v="):
+        raise InvalidYouTubeURL("could not extract a video ID from URL")
+    return normalized
+
+
+def normalize_youtube_url(url: str) -> str:
+    """Coerce a YouTube URL to a single-video form so yt-dlp doesn't end up in
+    the playlist extractor. Pass non-YouTube URLs through unchanged.
+
+    Cases handled:
+      * `watch?v=X&list=...` -> `watch?v=X` (drop the playlist context)
+      * `?list=RD<videoId>&start_radio=1` -> `watch?v=<videoId>` (Radio
+        playlists embed the seed in the list ID; YouTube refuses to view the
+        playlist directly with "This playlist type is unviewable.")
+      * `youtu.be/<videoId>` -> `watch?v=<videoId>`
+      * `youtube.com/shorts/<videoId>` -> `watch?v=<videoId>`
+    Everything else (PL/OL/algorithmic playlists with no derivable seed) is
+    left alone -- yt-dlp will surface its own error.
+    """
+    try:
+        parsed = urllib.parse.urlparse(url)
+    except Exception:
+        return url
+    host = (parsed.hostname or "").lower()
+    for prefix in ("www.", "m.", "music."):
+        if host.startswith(prefix):
+            host = host[len(prefix) :]
+            break
+    if host not in ("youtube.com", "youtu.be"):
+        return url
+
+    qs = urllib.parse.parse_qs(parsed.query)
+    if (vid := (qs.get("v") or [None])[0]) and _VIDEO_ID_RE.match(vid):
+        return f"https://www.youtube.com/watch?v={vid}"
+
+    if (
+        (lst := (qs.get("list") or [None])[0])
+        and lst.startswith("RD")
+        and _VIDEO_ID_RE.match(lst[2:13])
+    ):
+        return f"https://www.youtube.com/watch?v={lst[2:13]}"
+
+    if host == "youtu.be":
+        vid = parsed.path.lstrip("/")
+        if _VIDEO_ID_RE.match(vid):
+            return f"https://www.youtube.com/watch?v={vid}"
+
+    if host == "youtube.com" and parsed.path.startswith("/shorts/"):
+        vid = parsed.path[len("/shorts/") :].lstrip("/").split("/")[0]
+        if _VIDEO_ID_RE.match(vid):
+            return f"https://www.youtube.com/watch?v={vid}"
+
+    return url
+
+
+def _download_video_track(job: Job, url: str, job_dir: Path) -> None:
+    """Best-effort: download a video-only H.264/MP4 stream to video.mp4 for the
+    MP4 export (issue #219). The audio source is downloaded separately as
+    usual; this is a second, additive fetch so the audio pipeline is untouched.
+
+    Video-only MP4 needs no ffmpeg merge, so this can't break an audio-only job:
+    any failure (no progressive MP4 video, network error, unsupported codec) is
+    logged and swallowed, leaving has_video False. A cancel mid-download raises
+    JobCancelled, which the runner treats like any other cancellation.
+
+    Capped at VIDEO_MAX_HEIGHT to keep downloads reasonable -- a full song at
+    1080p is large, and the MP4 export doesn't need it."""
+
+    def vhook(d: dict) -> None:
+        if job.cancel_requested:
+            raise JobCancelled()
+        if d.get("status") == "downloading":
+            total = d.get("total_bytes") or d.get("total_bytes_estimate")
+            if total:
+                p = float(d.get("downloaded_bytes", 0)) / float(total)
+                _set(job, stage=f"Fetching video {int(p * 100)}%")
+
+    # Prefer H.264 (avc1) so the exported MP4 plays everywhere -- YouTube also
+    # serves AV1/VP9 in mp4 containers, which many players (Safari/iOS, older
+    # devices) can't decode. Fall back to any <=cap mp4 only if no avc1 exists.
+    max_height = get_video_max_height()
+    ydl_opts = {
+        "format": (
+            f"bestvideo[height<={max_height}][vcodec^=avc1]"
+            f"/bestvideo[height<={max_height}][ext=mp4]"
+        ),
+        "outtmpl": str(job_dir / "video.%(ext)s"),
+        "quiet": True,
+        "noprogress": True,
+        "noplaylist": True,
+        "allowed_extractors": _ALLOWED_EXTRACTORS,
+        "progress_hooks": [vhook],
+    }
+    # Point yt-dlp at the bundled ffmpeg in case a DASH stream needs remuxing;
+    # in portable builds ffmpeg is not on PATH.
+    if FFMPEG_DIR.is_dir():
+        ydl_opts["ffmpeg_location"] = str(FFMPEG_DIR)
+
+    _set(job, stage="Fetching video...")
+    try:
+        with YoutubeDL(ydl_opts) as ydl:
+            ydl.extract_info(url, download=True)
+    except JobCancelled:
+        raise
+    except Exception as exc:
+        if job.cancel_requested:
+            raise JobCancelled() from exc
+        logger.warning("[%s] video track unavailable (audio-only): %s", job.id, exc)
+
+    video = job_dir / "video.mp4"
+    if video.is_file() and video.stat().st_size > 0:
+        job.has_video = True
+    else:
+        # Drop any partial/non-mp4 leftover so the export endpoint sees nothing.
+        for f in job_dir.glob("video.*"):
+            f.unlink(missing_ok=True)
+
+
+def download(job: Job, url: str, job_dir: Path) -> Path:
+    url = normalize_youtube_url(url)
+    logger.info("[%s] download starting: %s", job.id, url)
+    _set(job, status="downloading", progress=0.0, stage="Processing...")
+
+    # Fetch metadata first (no download) so we can reject videos that are
+    # too long before wasting bandwidth and disk.
+    with YoutubeDL(
+        {"quiet": True, "noplaylist": True, "allowed_extractors": _ALLOWED_EXTRACTORS}
+    ) as ydl:
+        meta = ydl.extract_info(url, download=False) or {}
+    duration = meta.get("duration") or 0
+    max_duration = get_max_duration_sec()
+    if duration > max_duration:
+        mins = max_duration // 60
+        raise RuntimeError(f"Video is {int(duration // 60)} min -- limit is {mins} min")
+
+    def hook(d: dict) -> None:
+        # yt-dlp calls this on each chunk; raising here aborts the download.
+        # The runner unwraps yt-dlp's DownloadError and routes to JobCancelled.
+        if job.cancel_requested:
+            raise JobCancelled()
+        if d.get("status") == "downloading":
+            total = d.get("total_bytes") or d.get("total_bytes_estimate")
+            if total:
+                p = float(d.get("downloaded_bytes", 0)) / float(total)
+                _set(job, progress=p, stage=f"Downloading {int(p * 100)}%")
+        elif d.get("status") == "finished":
+            _set(job, progress=1.0, stage="Download complete")
+
+    # YouTube jobs additionally fetch the real video stream (below) for the
+    # MP4 export (issue #219). SoundCloud is audio-only and excluded.
+    is_youtube = url.startswith("https://www.youtube.com/")
+
+    # No postprocessors -- Demucs reads the raw audio container (webm/m4a/opus/...)
+    # directly via torchaudio + ffmpeg. Skipping the WAV transcode saves the slowest
+    # part of the download pipeline and a lot of disk.
+    ydl_opts = {
+        "format": "bestaudio/best",
+        "outtmpl": str(job_dir / "source.%(ext)s"),
+        "quiet": True,
+        "noprogress": True,
+        "noplaylist": True,
+        "allowed_extractors": _ALLOWED_EXTRACTORS,
+        "progress_hooks": [hook],
+    }
+    info: dict = {}
+    for attempt in range(_MAX_RETRIES + 1):
+        try:
+            with YoutubeDL(ydl_opts) as ydl:
+                info = ydl.extract_info(url, download=True) or {}
+            break
+        except Exception as exc:
+            if job.cancel_requested:
+                raise JobCancelled() from exc
+            if attempt < _MAX_RETRIES and _is_retriable(exc):
+                wait = _RETRY_BACKOFF[attempt]
+                logger.warning(
+                    "[%s] download attempt %d/%d failed (%s), retrying in %ds",
+                    job.id,
+                    attempt + 1,
+                    _MAX_RETRIES,
+                    exc,
+                    wait,
+                )
+                _set(job, stage=f"Network error — retrying ({attempt + 1}/{_MAX_RETRIES})...")
+                time.sleep(wait)
+            else:
+                raise
+
+    _set(
+        job,
+        title=info.get("title") or meta.get("title"),
+        duration_sec=info.get("duration") or duration,
+        thumbnail=info.get("thumbnail") or meta.get("thumbnail"),
+    )
+
+    raw_tags = [
+        t.strip().lower()
+        for t in (info.get("tags") or []) + (info.get("categories") or [])
+        if isinstance(t, str) and t.strip()
+    ]
+    seen: set[str] = set()
+    deduped = [t for t in raw_tags if not (t in seen or seen.add(t))]  # type: ignore[func-returns-value]
+    _set(job, tags=deduped[:8] or None)
+
+    # Best-effort: fetch the real video stream for the MP4 export.
+    # Non-fatal -- on any failure the job proceeds audio-only.
+    if is_youtube:
+        _download_video_track(job, url, job_dir)
+
+    candidates = sorted(job_dir.glob("source.*"))
+    if not candidates:
+        raise RuntimeError("yt-dlp finished but no source file was produced")
+    logger.info("[%s] download complete: %s", job.id, candidates[0].name)
+    return candidates[0]

app/pipeline/runner.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import shutil
+import subprocess
+from pathlib import Path
+
+from app.core.config import TIMEOUT_FFMPEG
+from app.core.models import Job, JobCancelled, _set
+from app.core.registry import persist as persist_registry
+from app.pipeline.analyze import analyze, compute_stem_presence
+from app.pipeline.collect import (
+    cleanup_source,
+    collect,
+    compute_stem_peaks,
+    make_original_track,
+    make_selected_mix,
+)
+from app.pipeline.download import download
+from app.pipeline.separate import separate
+
+logger = logging.getLogger("stemdeck.pipeline")
+
+
+def _rmtree(path: Path) -> None:
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError:
+        pass
+    except Exception:
+        logger.warning("failed to remove %s", path, exc_info=True)
+
+
+# Only one heavy job runs at a time -- Demucs is GPU/CPU-hungry.
+_pipeline_lock = asyncio.Semaphore(1)
+
+
+def _check_cancel(job: Job) -> None:
+    if job.cancel_requested:
+        raise JobCancelled()
+
+
+def _extract_video_track(job: Job, source: Path, job_dir: Path) -> None:
+    """For an .mp4 upload, preserve a silent video-only track at
+    video.mp4 so the studio can later mux it with a custom stem mix
+    into an MP4 (issue #219). Stream-copies the video (no
+    re-encode) -- fast and lossless.
+
+    Best-effort: an .mp4 with no video stream (audio-only container)
+    fails harmlessly and leaves has_video false."""
+    from app.core.config import ffmpeg_executable
+
+    dest = job_dir / "video.mp4"
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(source),
+        "-an",  # drop audio -- the mix is added at export time
+        "-c:v",
+        "copy",
+        "-movflags",
+        "+faststart",
+        "-y",
+        str(dest),
+    ]
+    result = subprocess.run(cmd, capture_output=True, timeout=TIMEOUT_FFMPEG)
+    if result.returncode != 0 or not dest.is_file() or dest.stat().st_size == 0:
+        dest.unlink(missing_ok=True)
+        logger.info("no video track preserved for job %s (source has no video stream?)", job.id)
+        return
+    job.has_video = True
+
+
+def _prepare_local_source(job: Job, source: Path, job_dir: Path) -> Path:
+    """Transcode any local upload to 16-bit 44.1 kHz stereo WAV before
+    handing it to Demucs. Normalises MP3 and non-standard WAV formats
+    (24-bit, 32-bit float, high sample rate, multi-channel) that Demucs
+    would otherwise process silently and output as silence.
+
+    For .mp4 uploads, first preserves a silent video.mp4 for later
+    MP4 export. Deletes the original source file after a
+    successful transcode."""
+    from app.core.config import ffmpeg_executable
+
+    dest = job_dir / "source.wav"
+    if source.resolve() == dest.resolve():
+        return source
+
+    _set(job, stage="Preparing audio...")
+    if source.suffix.lower() == ".mp4":
+        _extract_video_track(job, source, job_dir)
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(source),
+        "-ar",
+        "44100",
+        "-ac",
+        "2",
+        "-sample_fmt",
+        "s16",
+        "-y",
+        str(dest),
+    ]
+    result = subprocess.run(cmd, capture_output=True, timeout=TIMEOUT_FFMPEG)
+    if result.returncode != 0:
+        raise RuntimeError(
+            "ffmpeg transcode failed: " + result.stderr.decode("utf-8", errors="replace").strip()
+        )
+    source.unlink(missing_ok=True)
+    return dest
+
+
+def _run_common(job: Job, source: Path, job_dir: Path) -> None:
+    """Analyze → separate → collect → mix. Shared by both YouTube and local
+    upload pipelines after their respective source acquisition steps."""
+    _check_cancel(job)
+    analyze(job, source)
+    _check_cancel(job)
+    stems_root = separate(job, source, job_dir)
+    found = collect(job, stems_root, job_dir)
+    stems_dir = job_dir / "stems"
+    job.stem_presence = compute_stem_presence(stems_dir, found)
+    # Source (100-300 MB or the local upload) is no longer needed after
+    # collect; delete it before the ffmpeg amix steps in case scratch space
+    # is tight.
+    cleanup_source(job_dir)
+    job.stems = [{"name": name, "url": f"/api/jobs/{job.id}/stems/{name}.wav"} for name in found]
+    _check_cancel(job)
+    _set(job, stage="Mixing tracks...")
+    original_path = make_original_track(job, job_dir, stems_dir)
+    if original_path is not None:
+        job.stems.insert(
+            0,
+            {
+                "name": "original",
+                "url": f"/api/jobs/{job.id}/stems/original.wav",
+            },
+        )
+    _check_cancel(job)
+    mix_path = make_selected_mix(job, stems_dir, found)
+    if mix_path is not None:
+        job.mix_url = f"/api/jobs/{job.id}/stems/{mix_path.name}"
+    _check_cancel(job)
+
+    all_stem_names = [s["name"] for s in job.stems]
+    if mix_path is not None and mix_path.stem not in all_stem_names:
+        all_stem_names.append(mix_path.stem)
+    compute_stem_peaks(stems_dir, all_stem_names)
+
+
+def _run_blocking(job: Job, url: str, job_dir: Path) -> None:
+    _check_cancel(job)
+    source = download(job, url, job_dir)
+    _run_common(job, source, job_dir)
+
+
+def _run_local_blocking(job: Job, source_path: Path, job_dir: Path) -> None:
+    _check_cancel(job)
+    source = _prepare_local_source(job, source_path, job_dir)
+    _run_common(job, source, job_dir)
+
+
+def _write_metadata(job: Job, job_dir: Path) -> None:
+    meta = {
+        "title": job.title,
+        "thumbnail": job.thumbnail,
+        "duration_sec": job.duration_sec,
+        "bpm": job.bpm,
+        "key": job.key,
+        "scale": job.scale,
+        "key_confidence": job.key_confidence,
+        "lufs": job.lufs,
+        "peak_db": job.peak_db,
+        "dynamic_range": job.dynamic_range,
+        "tempo_stability": job.tempo_stability,
+        "stem_presence": job.stem_presence,
+        "tags": job.tags,
+        "has_video": job.has_video,
+    }
+    try:
+        (job_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8")
+    except OSError:
+        logger.warning("could not write metadata.json for job %s", job.id, exc_info=True)
+
+
+async def _run_async(
+    job: Job,
+    job_dir: Path,
+    jobs_dir: Path,
+    blocking_fn,
+    *fn_args: object,
+    error_msg: str = "Audio processing failed. Please try again.",
+) -> None:
+    """Common async wrapper: acquires the pipeline lock, runs blocking_fn in a
+    thread, then handles success / cancel / error outcomes uniformly."""
+    try:
+        async with _pipeline_lock:
+            await asyncio.to_thread(blocking_fn, job, *fn_args, job_dir)
+    except Exception as e:
+        if not isinstance(e, JobCancelled) and not job.cancel_requested:
+            logger.exception("pipeline failed for job %s: %s", job.id, e)
+            _set(job, status="error", stage="Error: Processing failed", error=error_msg)
+            persist_registry(jobs_dir)
+            _rmtree(job_dir)
+            return
+        logger.info(
+            "pipeline cancelled%s for job %s",
+            " (wrapped)" if not isinstance(e, JobCancelled) else "",
+            job.id,
+        )
+        _set(job, status="cancelled", stage="Cancelled")
+        persist_registry(jobs_dir)
+        _rmtree(job_dir)
+        return
+    _set(job, status="done", progress=1.0, stage="Done")
+    _write_metadata(job, job_dir)
+    persist_registry(jobs_dir)
+
+
+async def run_pipeline(job: Job, url: str, jobs_dir: Path) -> None:
+    job_dir = jobs_dir / job.id
+    try:
+        job_dir.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        logger.exception("pipeline failed for job %s: %s", job.id, e)
+        _set(
+            job,
+            status="error",
+            stage="Error: Processing failed",
+            error="Audio processing failed. Please try another video.",
+        )
+        persist_registry(jobs_dir)
+        return
+    await _run_async(
+        job,
+        job_dir,
+        jobs_dir,
+        _run_blocking,
+        url,
+        error_msg="Audio processing failed. Please try another video.",
+    )
+
+
+async def run_local_pipeline(job: Job, source_path: Path, jobs_dir: Path) -> None:
+    """Run the stem-separation pipeline for a locally uploaded file.
+    The job directory and source file are already present on disk (created
+    by the API handler before this task is scheduled)."""
+    job_dir = jobs_dir / job.id
+    await _run_async(job, job_dir, jobs_dir, _run_local_blocking, source_path)

app/pipeline/separate.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import logging
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+from app.core.config import DEMUCS_DEVICE, DEMUCS_MODEL, TIMEOUT_DEMUCS_STALL
+from app.core.models import Job, JobCancelled, _set
+from app.core.registry import set_proc
+
+logger = logging.getLogger("stemdeck.pipeline")
+
+_PCT_RE = re.compile(r"(\d{1,3})%")
+# Terminate demucs if stderr produces no output for this many seconds.
+# GPU processing can be silent for minutes; 30 min covers legitimate pauses
+# while still catching genuine hangs (GPU deadlock, OOM stall, etc.).
+
+
+def separate(job: Job, source: Path, job_dir: Path) -> Path:
+    _set(job, status="separating", progress=0.0, stage="Separating stems...")
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "demucs",
+        "-n",
+        DEMUCS_MODEL,
+        "-d",
+        DEMUCS_DEVICE,
+        "-o",
+        str(job_dir),
+        str(source),
+    ]
+    env = os.environ.copy()
+    try:
+        import certifi
+
+        env.setdefault("SSL_CERT_FILE", certifi.where())
+        env.setdefault("REQUESTS_CA_BUNDLE", certifi.where())
+    except ModuleNotFoundError:
+        pass
+
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=0,
+        env=env,
+    )
+    if proc.stderr is None:
+        raise RuntimeError("demucs subprocess has no stderr pipe")
+    set_proc(job.id, proc)
+
+    # tqdm uses \r to redraw -- read char-by-char and split on \r or \n.
+    # Keep the last few non-progress lines so we can surface them if demucs
+    # exits non-zero (otherwise the only signal would be a bare exit code).
+    buf = ""
+    tail: list[str] = []
+    last_output: list[float] = [time.monotonic()]
+    # Event set by the reader loop when the process exits normally so the
+    # watchdog can wake up immediately instead of waiting out its 30 s sleep.
+    _done_evt = threading.Event()
+
+    def _watchdog() -> None:
+        while not _done_evt.wait(timeout=30):
+            if proc.poll() is not None:
+                return
+            if time.monotonic() - last_output[0] > TIMEOUT_DEMUCS_STALL:
+                logger.warning(
+                    "demucs stalled for %ss with no output, terminating job %s",
+                    TIMEOUT_DEMUCS_STALL,
+                    job.id,
+                )
+                proc.terminate()
+                return
+
+    wt = threading.Thread(target=_watchdog, daemon=True)
+    wt.start()
+    try:
+        while True:
+            ch = proc.stderr.read(1)
+            if not ch:
+                break
+            last_output[0] = time.monotonic()
+            if ch in ("\r", "\n"):
+                line = buf.strip()
+                buf = ""
+                if not line:
+                    continue
+                m = _PCT_RE.search(line)
+                if m:
+                    pct = max(0, min(100, int(m.group(1))))
+                    _set(job, progress=pct / 100.0, stage=f"Separating {pct}%")
+                else:
+                    tail.append(line)
+                    if len(tail) > 40:
+                        tail.pop(0)
+            else:
+                buf += ch
+
+        proc.wait()
+    finally:
+        _done_evt.set()
+        set_proc(job.id, None)
+        wt.join(timeout=2)
+
+    # POST /cancel calls proc.terminate() directly, which causes the read loop
+    # above to hit EOF and proc.wait() to return a nonzero status. Translate
+    # that into JobCancelled before the generic "demucs failed" path.
+    if job.cancel_requested:
+        raise JobCancelled()
+    if proc.returncode != 0:
+        detail = "\n".join(tail[-15:]) if tail else "(no stderr captured)"
+        logger.error("[%s] demucs exited %s; tail:\n%s", job.id, proc.returncode, detail)
+        last = tail[-1] if tail else f"exit status {proc.returncode}"
+        raise RuntimeError(f"demucs failed: {last}")
+
+    stems_root = job_dir / DEMUCS_MODEL / source.stem
+    if not stems_root.is_dir():
+        raise RuntimeError(f"demucs output not found at {stems_root}")
+    return stems_root