chore: import upstream snapshot with attribution

2026-07-13 12:32:09 +08:00
commit 8f10353f0c
135 changed files with 37786 additions and 0 deletions
@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import logging
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+from app.core.config import DEMUCS_DEVICE, DEMUCS_MODEL, TIMEOUT_DEMUCS_STALL
+from app.core.models import Job, JobCancelled, _set
+from app.core.registry import set_proc
+
+logger = logging.getLogger("stemdeck.pipeline")
+
+_PCT_RE = re.compile(r"(\d{1,3})%")
+# Terminate demucs if stderr produces no output for this many seconds.
+# GPU processing can be silent for minutes; 30 min covers legitimate pauses
+# while still catching genuine hangs (GPU deadlock, OOM stall, etc.).
+
+
+def separate(job: Job, source: Path, job_dir: Path) -> Path:
+    _set(job, status="separating", progress=0.0, stage="Separating stems...")
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "demucs",
+        "-n",
+        DEMUCS_MODEL,
+        "-d",
+        DEMUCS_DEVICE,
+        "-o",
+        str(job_dir),
+        str(source),
+    ]
+    env = os.environ.copy()
+    try:
+        import certifi
+
+        env.setdefault("SSL_CERT_FILE", certifi.where())
+        env.setdefault("REQUESTS_CA_BUNDLE", certifi.where())
+    except ModuleNotFoundError:
+        pass
+
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=0,
+        env=env,
+    )
+    if proc.stderr is None:
+        raise RuntimeError("demucs subprocess has no stderr pipe")
+    set_proc(job.id, proc)
+
+    # tqdm uses \r to redraw -- read char-by-char and split on \r or \n.
+    # Keep the last few non-progress lines so we can surface them if demucs
+    # exits non-zero (otherwise the only signal would be a bare exit code).
+    buf = ""
+    tail: list[str] = []
+    last_output: list[float] = [time.monotonic()]
+    # Event set by the reader loop when the process exits normally so the
+    # watchdog can wake up immediately instead of waiting out its 30 s sleep.
+    _done_evt = threading.Event()
+
+    def _watchdog() -> None:
+        while not _done_evt.wait(timeout=30):
+            if proc.poll() is not None:
+                return
+            if time.monotonic() - last_output[0] > TIMEOUT_DEMUCS_STALL:
+                logger.warning(
+                    "demucs stalled for %ss with no output, terminating job %s",
+                    TIMEOUT_DEMUCS_STALL,
+                    job.id,
+                )
+                proc.terminate()
+                return
+
+    wt = threading.Thread(target=_watchdog, daemon=True)
+    wt.start()
+    try:
+        while True:
+            ch = proc.stderr.read(1)
+            if not ch:
+                break
+            last_output[0] = time.monotonic()
+            if ch in ("\r", "\n"):
+                line = buf.strip()
+                buf = ""
+                if not line:
+                    continue
+                m = _PCT_RE.search(line)
+                if m:
+                    pct = max(0, min(100, int(m.group(1))))
+                    _set(job, progress=pct / 100.0, stage=f"Separating {pct}%")
+                else:
+                    tail.append(line)
+                    if len(tail) > 40:
+                        tail.pop(0)
+            else:
+                buf += ch
+
+        proc.wait()
+    finally:
+        _done_evt.set()
+        set_proc(job.id, None)
+        wt.join(timeout=2)
+
+    # POST /cancel calls proc.terminate() directly, which causes the read loop
+    # above to hit EOF and proc.wait() to return a nonzero status. Translate
+    # that into JobCancelled before the generic "demucs failed" path.
+    if job.cancel_requested:
+        raise JobCancelled()
+    if proc.returncode != 0:
+        detail = "\n".join(tail[-15:]) if tail else "(no stderr captured)"
+        logger.error("[%s] demucs exited %s; tail:\n%s", job.id, proc.returncode, detail)
+        last = tail[-1] if tail else f"exit status {proc.returncode}"
+        raise RuntimeError(f"demucs failed: {last}")
+
+    stems_root = job_dir / DEMUCS_MODEL / source.stem
+    if not stems_root.is_dir():
+        raise RuntimeError(f"demucs output not found at {stems_root}")
+    return stems_root