chore: import upstream snapshot with attribution

scripts/analyze_marqs.mjs

@@ -0,0 +1,146 @@
+#!/usr/bin/env node
+
+const filename = process.argv[2];
+
+if (!filename) {
+  console.error("Usage: analyze_marqs.mjs <filename>");
+  process.exit(1);
+}
+
+import fs from "fs/promises";
+
+(async () => {
+  try {
+    const input = await fs.readFile(filename, "utf-8");
+
+    await processInput(input);
+  } catch (err) {
+    console.error(`Error reading file: ${err}`);
+    process.exit(1);
+  }
+})();
+
+// input is jsonl format, we want to split by line and then JSON parse each line
+async function processInput(input) {
+  const rows = [];
+  const lines = input.split("\n");
+  for (const line of lines) {
+    if (!line) {
+      continue;
+    }
+
+    const row = JSON.parse(line);
+
+    // process each row
+    rows.push(row);
+  }
+
+  const queueChoiceCounts = {};
+  const queueMaxAges = {};
+  const queueMaxSizes = {};
+  const nextRangeOffsetCounts = {};
+  const consumerStats = {};
+  const rowsByConsumer = {};
+
+  console.log(`Processed ${rows.length} rows`);
+
+  // console.log(util.inspect(rows[0], { depth: 20 }));
+
+  for (const row of rows) {
+    const queueChoice = row.queueChoice;
+
+    if (queueChoice) {
+      if (!queueChoiceCounts[queueChoice]) {
+        queueChoiceCounts[queueChoice] = 0;
+      }
+      queueChoiceCounts[queueChoice]++;
+    }
+  }
+
+  for (const row of rows) {
+    rowsByConsumer[row.consumerId] = rowsByConsumer[row.consumerId] || [];
+    rowsByConsumer[row.consumerId].push(row);
+
+    const queueChoice = row.queueChoice;
+
+    if (!consumerStats[row.consumerId]) {
+      consumerStats[row.consumerId] = {
+        queueChoiceCounts: {},
+        totalQueueChoices: 0,
+        noQueueChoiceCount: 0,
+      };
+    }
+
+    if (queueChoice) {
+      const queueData = row.queuesWithScores.find((queue) => queue.queue === queueChoice);
+
+      console.log(
+        `[${row.timestamp}] -> ${queueChoice} [age:${queueData.age}] [size:${queueData.size}] [nextRange.offset=${row.nextRange.offset}] [queuesWithScores=${row.queuesWithScores.length}] [${row.consumerId}]`
+      );
+
+      if (!queueMaxAges[queueChoice] || queueData.age > queueMaxAges[queueChoice]) {
+        queueMaxAges[queueChoice] = queueData.age;
+      }
+
+      if (!queueMaxSizes[queueChoice] || queueData.size > queueMaxSizes[queueChoice]) {
+        queueMaxSizes[queueChoice] = queueData.size;
+      }
+
+      if (!consumerStats[row.consumerId].queueChoiceCounts[queueChoice]) {
+        consumerStats[row.consumerId].queueChoiceCounts[queueChoice] = 0;
+      }
+
+      consumerStats[row.consumerId].queueChoiceCounts[queueChoice]++;
+      consumerStats[row.consumerId].totalQueueChoices++;
+    } else {
+      console.log(
+        `[${row.timestamp}] -> No queue choice [nextRange.offset=${row.nextRange.offset}] [queuesWithScores=${row.queuesWithScores.length}] [${row.consumerId}]`
+      );
+
+      consumerStats[row.consumerId].noQueueChoiceCount++;
+    }
+
+    if (!nextRangeOffsetCounts[row.nextRange.offset]) {
+      nextRangeOffsetCounts[row.nextRange.offset] = 0;
+    }
+
+    nextRangeOffsetCounts[row.nextRange.offset]++;
+  }
+
+  console.log("Queue choice counts:");
+  console.log(queueChoiceCounts);
+
+  console.log("Queue max ages:");
+  console.log(queueMaxAges);
+
+  console.log("Queue max sizes:");
+  console.log(queueMaxSizes);
+
+  console.log("Next range offset counts:");
+  console.log(nextRangeOffsetCounts);
+
+  for (const consumerId in consumerStats) {
+    console.log(`Consumer ${consumerId}:`);
+    console.log(consumerStats[consumerId]);
+  }
+
+  for (const consumerId in rowsByConsumer) {
+    console.log(`\n## Consumer ${consumerId}:`);
+
+    for (const row of rowsByConsumer[consumerId]) {
+      const queueChoice = row.queueChoice;
+
+      if (queueChoice) {
+        const queueData = row.queuesWithScores.find((queue) => queue.queue === queueChoice);
+
+        console.log(
+          `[${row.timestamp}] -> ${queueChoice} [age:${queueData.age}] [size:${queueData.size}] [nextRange.offset=${row.nextRange.offset}] [queuesWithScores=${row.queuesWithScores.length}] [${row.consumerId}]`
+        );
+      } else {
+        console.log(
+          `[${row.timestamp}] -> No queue choice [nextRange.offset=${row.nextRange.offset}] [queuesWithScores=${row.queuesWithScores.length}] [${row.consumerId}]`
+        );
+      }
+    }
+  }
+}

scripts/batch-concurrency-cleaner.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+#
+# Batch Queue Concurrency Cleaner
+#
+# Detects and cleans up stale concurrency entries that block batch processing.
+# This is a workaround for a bug where visibility timeout reclaims don't release concurrency.
+#
+# Uses a Lua script for ATOMIC detection of stale entries - no race conditions.
+#
+# Usage:
+#   ./batch-concurrency-cleaner.sh --read-redis <url> --write-redis <url> [--delay <seconds>] [--dry-run]
+#
+
+set -e
+
+# Defaults
+DELAY=10
+DRY_RUN=false
+READ_REDIS=""
+WRITE_REDIS=""
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --read-redis)
+      READ_REDIS="$2"
+      shift 2
+      ;;
+    --write-redis)
+      WRITE_REDIS="$2"
+      shift 2
+      ;;
+    --delay)
+      DELAY="$2"
+      shift 2
+      ;;
+    --dry-run)
+      DRY_RUN=true
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: $0 --read-redis <url> --write-redis <url> [--delay <seconds>] [--dry-run]"
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "$READ_REDIS" ]] || [[ -z "$WRITE_REDIS" ]]; then
+  echo "Error: --read-redis and --write-redis are required"
+  exit 1
+fi
+
+echo "Batch Queue Concurrency Cleaner (Atomic Version)"
+echo "================================================="
+echo "Read Redis:  ${READ_REDIS:0:30}..."
+echo "Write Redis: ${WRITE_REDIS:0:30}..."
+echo "Delay:       ${DELAY}s"
+echo "Dry run:     $DRY_RUN"
+echo ""
+
+rcli_read() {
+  redis-cli -u "$READ_REDIS" --no-auth-warning "$@" 2>/dev/null
+}
+
+rcli_write() {
+  redis-cli -u "$WRITE_REDIS" --no-auth-warning "$@" 2>/dev/null
+}
+
+# Lua script that ATOMICALLY checks for stale concurrency entries
+# KEYS[1] = concurrency key to check
+# KEYS[2-13] = in-flight data hash keys for shards 0-11
+# Returns: list of stale messageIds (not in any in-flight hash)
+FIND_STALE_LUA='
+local concurrency_key = KEYS[1]
+local stale = {}
+
+-- Get all members of the concurrency set
+local members = redis.call("SMEMBERS", concurrency_key)
+
+for _, msg_id in ipairs(members) do
+  local found = false
+  -- Check each in-flight shard (KEYS[2] through KEYS[13])
+  for i = 2, 13 do
+    if redis.call("HEXISTS", KEYS[i], msg_id) == 1 then
+      found = true
+      break
+    end
+  end
+  if not found then
+    table.insert(stale, msg_id)
+  end
+end
+
+return stale
+'
+
+# Build the in-flight keys array (used in every Lua call)
+INFLIGHT_KEYS="engine:batch:inflight:0:data"
+for shard in 1 2 3 4 5 6 7 8 9 10 11; do
+  INFLIGHT_KEYS="$INFLIGHT_KEYS engine:batch:inflight:$shard:data"
+done
+
+# Main loop
+while true; do
+  ts=$(date '+%H:%M:%S')
+
+  # Get master queue total and in-flight count for status display
+  master_total=0
+  for i in 0 1 2 3 4 5 6 7 8 9 10 11; do
+    count=$(rcli_read ZCARD "engine:batch:master:$i")
+    master_total=$((master_total + count))
+  done
+
+  inflight_total=0
+  for i in 0 1 2 3 4 5 6 7 8 9 10 11; do
+    count=$(rcli_read HLEN "engine:batch:inflight:$i:data")
+    inflight_total=$((inflight_total + count))
+  done
+
+  # Scan for concurrency keys
+  cursor=0
+  total_stale=0
+  cleaned_tenants=0
+
+  while true; do
+    scan_output=$(rcli_read SCAN $cursor MATCH 'engine:batch:concurrency:tenant:*' COUNT 1000)
+    cursor=$(echo "$scan_output" | head -1)
+    keys=$(echo "$scan_output" | tail -n +2)
+
+    while IFS= read -r conc_key; do
+      [[ -z "$conc_key" ]] && continue
+
+      # ATOMIC check: Run Lua script to find stale entries
+      # 13 keys total: 1 concurrency key + 12 in-flight keys
+      stale_ids=$(rcli_read EVAL "$FIND_STALE_LUA" 13 "$conc_key" $INFLIGHT_KEYS)
+
+      # Count stale entries
+      stale_count=0
+      stale_array=()
+      while IFS= read -r stale_id; do
+        [[ -z "$stale_id" ]] && continue
+        stale_array+=("$stale_id")
+        stale_count=$((stale_count + 1))
+      done <<< "$stale_ids"
+
+      if [[ $stale_count -gt 0 ]]; then
+        tenant="${conc_key#engine:batch:concurrency:tenant:}"
+        total_stale=$((total_stale + stale_count))
+
+        if [[ "$DRY_RUN" == "true" ]]; then
+          echo "[$ts] STALE (dry-run): $tenant ($stale_count entries)"
+          for sid in "${stale_array[@]}"; do
+            echo "       - $sid"
+          done
+        else
+          # Remove each stale entry individually with SREM (idempotent, safe)
+          for sid in "${stale_array[@]}"; do
+            rcli_write SREM "$conc_key" "$sid" >/dev/null
+          done
+          echo "[$ts] CLEANED: $tenant ($stale_count stale entries removed)"
+          cleaned_tenants=$((cleaned_tenants + 1))
+        fi
+      fi
+    done <<< "$keys"
+
+    [[ "$cursor" == "0" ]] && break
+  done
+
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "[$ts] in-flight=$inflight_total master-queue=$master_total stale-found=$total_stale"
+  else
+    echo "[$ts] in-flight=$inflight_total master-queue=$master_total cleaned=$cleaned_tenants"
+  fi
+
+  sleep "$DELAY"
+done

scripts/build-dockerfile.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+docker build -t local-triggerdotdev:latest -f docker/Dockerfile .
+image=local-triggerdotdev:latest
+src=/triggerdotdev
+dst=$(mktemp -d)
+
+mkdir -p $dst
+
+echo -e "Extracting image into $dst..."
+
+container=$(docker create "$image")
+docker cp "$container:$src" "$dst"
+docker rm "$container"
+/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code "$dst/triggerdotdev"
+

scripts/bump-helm-chart.mjs

@@ -0,0 +1,31 @@
+import { readFileSync, writeFileSync } from "node:fs";
+
+const VERSION_SOURCE = "packages/cli-v3/package.json";
+const CHART_PATH = "hosting/k8s/helm/Chart.yaml";
+
+const { version } = JSON.parse(readFileSync(VERSION_SOURCE, "utf8"));
+const desiredVersion = `version: ${version}`;
+const desiredAppVersion = `appVersion: v${version}`;
+
+const original = readFileSync(CHART_PATH, "utf8");
+
+const versionMatch = original.match(/^version:.*$/m);
+const appVersionMatch = original.match(/^appVersion:.*$/m);
+
+if (!versionMatch || !appVersionMatch) {
+  const missing = [!versionMatch && "version:", !appVersionMatch && "appVersion:"]
+    .filter(Boolean)
+    .join(", ");
+  console.error(`${CHART_PATH} is missing required key(s): ${missing}`);
+  process.exit(1);
+}
+
+if (versionMatch[0] === desiredVersion && appVersionMatch[0] === desiredAppVersion) {
+  console.log(`${CHART_PATH} already at ${version} (from ${VERSION_SOURCE}), no changes`);
+} else {
+  const updated = original
+    .replace(/^version:.*/m, desiredVersion)
+    .replace(/^appVersion:.*/m, desiredAppVersion);
+  writeFileSync(CHART_PATH, updated);
+  console.log(`${CHART_PATH} bumped to ${version} (from ${VERSION_SOURCE})`);
+}

scripts/bundleSdkDocs.ts

@@ -0,0 +1,133 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+
+// Snapshots the user-facing docs into the SDK package, so AI coding agents can read the
+// version-pinned reference directly from node_modules (zero drift). Run as part of
+// `@trigger.dev/sdk`'s build, from the package dir.
+//
+// The manifest is the entire "Documentation" dropdown in `docs/docs.json` (the
+// "Resources for Trigger.dev" tab) — every page under it is bundled. Add a page to that
+// nav and it ships automatically; nothing else to edit. The API reference and
+// Guides & examples dropdowns are intentionally not bundled. Skills reference into this
+// set by path; their `sources:` frontmatter is informational and no longer drives bundling.
+//
+// Layout: nav page `tasks/overview` is copied from `docs/tasks/overview.mdx` to
+// `<sdk>/docs/tasks/overview.mdx`, so a skill at `<sdk>/skills/<name>/SKILL.md` reaches it
+// at `../../docs/tasks/overview.mdx` and an agent reaches it at `@trigger.dev/sdk/docs/...`.
+
+const packageDir = process.cwd(); // packages/trigger-sdk when run from the SDK build
+const repoRoot = path.resolve(packageDir, "..", "..");
+const docsRoot = path.join(repoRoot, "docs");
+const outDir = path.join(packageDir, "docs");
+
+const DROPDOWN = "Documentation";
+
+/** Recursively collect every page path under a docs.json nav node (groups -> pages, nested). */
+function collectPages(node: unknown): string[] {
+  const out: string[] = [];
+  if (node && typeof node === "object") {
+    const n = node as { groups?: unknown[]; pages?: unknown[] };
+    for (const g of n.groups ?? []) out.push(...collectPages(g));
+    for (const p of n.pages ?? []) {
+      if (typeof p === "string") out.push(p);
+      else out.push(...collectPages(p));
+    }
+  }
+  return out;
+}
+
+async function collectManifest(): Promise<string[]> {
+  const docsJson = JSON.parse(await fs.readFile(path.join(docsRoot, "docs.json"), "utf8"));
+  const dropdowns: Array<{ dropdown?: string }> = docsJson?.navigation?.dropdowns ?? [];
+  const documentation = dropdowns.find((d) => d.dropdown === DROPDOWN);
+
+  if (!documentation) {
+    throw new Error(`[bundleSdkDocs] "${DROPDOWN}" dropdown not found in docs/docs.json`);
+  }
+
+  // Page paths are root-relative without extension (e.g. "tasks/overview"); map to docs/*.mdx.
+  return [...new Set(collectPages(documentation))];
+}
+
+async function bundleSdkDocs() {
+  // When the SDK is built as a dependency inside a pruned workspace (e.g. the webapp Docker
+  // image), the repo-level docs/ tree is a separate workspace package that isn't part of that
+  // build's dependency graph, so it isn't present. The SDK isn't being published there, so
+  // there's nothing to bundle: skip rather than fail. Publishing always runs from the full
+  // monorepo where docs/ exists, so the guards below still protect releases.
+  try {
+    await fs.access(docsRoot);
+  } catch {
+    console.log(`[bundleSdkDocs] docs/ not present at ${docsRoot}; skipping (pruned build)`);
+    return;
+  }
+
+  const manifest = await collectManifest();
+
+  if (manifest.length === 0) {
+    // The nav structure changed shape; refuse to ship the SDK with no docs.
+    throw new Error(`[bundleSdkDocs] no pages found under the "${DROPDOWN}" dropdown`);
+  }
+
+  // Rebuild from scratch so removed pages don't linger in the package.
+  await fs.rm(outDir, { recursive: true, force: true });
+
+  const missing: string[] = [];
+  let copied = 0;
+
+  for (const rel of manifest) {
+    // Defensive: nav paths come from our own docs.json and are URL-style, but a
+    // fat-fingered `../`, a backslash, or an absolute path shouldn't be able to copy a
+    // file from outside docs/ into the package. Reject backslashes (Windows separator)
+    // and both POSIX and Windows absolute forms, then the normalized `..` traversal.
+    const safeRel = path.posix.normalize(rel);
+    if (
+      rel.includes("\\") ||
+      path.posix.isAbsolute(rel) ||
+      path.win32.isAbsolute(rel) ||
+      safeRel.startsWith("..")
+    ) {
+      throw new Error(`[bundleSdkDocs] invalid nav path "${rel}" under "${DROPDOWN}"`);
+    }
+
+    const src = path.join(docsRoot, `${safeRel}.mdx`);
+    try {
+      await fs.access(src);
+    } catch {
+      // A nav entry pointing at a nonexistent page is a docs-nav issue, not a bundler one.
+      // Warn and skip rather than fail the SDK build.
+      missing.push(rel);
+      continue;
+    }
+    const dest = path.join(outDir, `${safeRel}.mdx`);
+    await fs.mkdir(path.dirname(dest), { recursive: true });
+    await fs.copyFile(src, dest);
+    copied++;
+  }
+
+  if (missing.length > 0) {
+    console.warn(
+      `[bundleSdkDocs] ${missing.length} "${DROPDOWN}" nav page(s) have no .mdx and were skipped:\n` +
+        missing.map((m) => `  - ${m}`).join("\n")
+    );
+  }
+
+  if (copied === 0) {
+    // Every nav page was missing on disk; refuse to ship the SDK with an empty docs bundle.
+    throw new Error(
+      `[bundleSdkDocs] 0 docs copied from the "${DROPDOWN}" nav; refusing empty docs bundle`
+    );
+  }
+
+  console.log(
+    `[bundleSdkDocs] bundled ${copied} docs from the "${DROPDOWN}" nav into ${path.relative(
+      repoRoot,
+      outDir
+    )}`
+  );
+}
+
+bundleSdkDocs().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

scripts/cleanup-server-changes.mjs

@@ -0,0 +1,18 @@
+import { readdirSync, unlinkSync } from "node:fs";
+
+const DIR = ".server-changes";
+const KEEP = new Set(["README.md"]);
+
+const removed = [];
+for (const file of readdirSync(DIR)) {
+  if (!file.endsWith(".md") || KEEP.has(file)) continue;
+  unlinkSync(`${DIR}/${file}`);
+  removed.push(file);
+}
+
+if (removed.length === 0) {
+  console.log(`${DIR} already clean, no changes`);
+} else {
+  console.log(`${DIR} cleaned, removed ${removed.length} consumed file(s):`);
+  removed.forEach((f) => console.log(`  - ${f}`));
+}

scripts/docker.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+// Cross-platform wrapper for `docker compose` that conditionally passes
+// `--env-file <repo-root>/.env` when the file exists. Replaces an earlier
+// inline `$([ -f .env ] && echo --env-file .env)` shell substitution that
+// only worked in POSIX shells, breaking native Windows `cmd.exe` runs.
+//
+// Used by the root `pnpm run docker` / `docker:full` scripts and by the
+// clickhouse package's `db:migrate` script. Always runs compose with cwd
+// set to the repo root, so callers can pass `-f docker/docker-compose.yml`
+// from anywhere in the workspace.
+import { existsSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const envPath = resolve(repoRoot, ".env");
+const envArgs = existsSync(envPath) ? ["--env-file", envPath] : [];
+
+try {
+  execFileSync("docker", ["compose", ...envArgs, ...process.argv.slice(2)], {
+    stdio: "inherit",
+    cwd: repoRoot,
+  });
+} catch (err) {
+  process.exit(err.status ?? 1);
+}

scripts/enhance-release-pr.mjs

@@ -0,0 +1,452 @@
+#!/usr/bin/env node
+
+/**
+ * Enhances the changeset release PR with a well-written, deduplicated summary.
+ *
+ * Reads:
+ *   - The raw changeset PR body (via CHANGESET_PR_BODY env var or stdin)
+ *   - .server-changes/*.md files for server-only changes
+ *
+ * Outputs a formatted PR body to stdout that includes:
+ *   - A clean summary with categories
+ *   - Server changes section
+ *   - The raw changeset output in a collapsed <details> section
+ *
+ * Usage:
+ *   CHANGESET_PR_BODY="..." node scripts/enhance-release-pr.mjs <version>
+ *   echo "$PR_BODY" | node scripts/enhance-release-pr.mjs <version>
+ */
+
+import { promises as fs } from "fs";
+import { execFile } from "child_process";
+import { join } from "path";
+
+const version = process.argv[2];
+if (!version) {
+  console.error("Usage: node scripts/enhance-release-pr.mjs <version>");
+  process.exit(1);
+}
+
+const ROOT_DIR = join(import.meta.dirname, "..");
+
+// --- Parse changeset PR body ---
+
+function parsePrBody(body) {
+  const entries = [];
+  if (!body) return entries;
+
+  // Deduplicate by entry content. A single changeset that targets multiple
+  // packages is rendered once per package section, so the same text repeats and
+  // we collapse it. But several distinct changesets from one PR have distinct
+  // text (and each still carries that PR's link), so keying on content keeps
+  // them all instead of dropping every entry after the first for that PR.
+  const seen = new Set();
+
+  // A standalone dependency-bump list item, e.g. "`@trigger.dev/core@4.5.0-rc.7`"
+  // or "trigger.dev@4.5.0-rc.7". These normally appear nested under
+  // "Updated dependencies:" (and so get swallowed into that item below), but we
+  // guard against them showing up on their own too. Crucially this only matches
+  // a line that is *entirely* a package bump, so a real changeset that merely
+  // begins with a package name (e.g. "`@trigger.dev/sdk` now bundles ...") is
+  // kept.
+  const depBumpPattern = /^`?(?:@trigger\.dev\/[\w-]+|trigger\.dev)@[\w.+-]+`?$/;
+
+  // Group lines into top-level list items. A top-level item starts with a bullet
+  // at column 0 ("- " / "* "); every indented or blank line below it (sub-bullets,
+  // fenced code blocks, continuation paragraphs) belongs to that same item.
+  const items = [];
+  let current = null;
+
+  const flush = () => {
+    if (!current) return;
+    while (current.length > 1 && current[current.length - 1].trim() === "") {
+      current.pop();
+    }
+    items.push(current);
+    current = null;
+  };
+
+  for (const line of body.split("\n")) {
+    const isTopLevelBullet = /^[-*]\s+/.test(line);
+    if (isTopLevelBullet) {
+      flush();
+      current = [line];
+    } else if (current) {
+      if (line.trim() === "" || /^\s/.test(line)) {
+        current.push(line);
+      } else {
+        // A non-indented, non-blank, non-bullet line (heading or prose) ends the item
+        flush();
+      }
+    }
+  }
+  flush();
+
+  for (const itemLines of items) {
+    const headLine = itemLines[0].replace(/^[-*]\s+/, "").trim();
+    if (!headLine) continue;
+
+    // Skip dependency-only updates
+    if (headLine.startsWith("Updated dependencies")) continue;
+    if (depBumpPattern.test(headLine)) continue;
+
+    // Reconstruct the full item: head line + dedented continuation lines, so
+    // code blocks and sub-bullets survive. Continuation under a "-   " item is
+    // indented 4 spaces; strip up to 4 to bring it back to the base level.
+    const continuation = itemLines.slice(1).map((l) => l.replace(/^ {1,4}/, ""));
+    const text = [headLine, ...continuation].join("\n").replace(/\s+$/, "");
+
+    // Deduplicate on the full entry text (which embeds the PR link). The same
+    // changeset echoed across package sections collapses to one, while multiple
+    // distinct changesets from a single PR are each preserved.
+    const dedupeKey = text.replace(/\s+/g, " ").trim();
+    if (seen.has(dedupeKey)) continue;
+    seen.add(dedupeKey);
+
+    // Categorize off the head line
+    const lower = headLine.toLowerCase();
+    let type = "improvement";
+    if (lower.startsWith("fix") || lower.includes("bug fix")) {
+      type = "fix";
+    } else if (
+      lower.startsWith("feat") ||
+      lower.includes("new feature") ||
+      lower.includes("add support") ||
+      lower.includes("added support") ||
+      lower.includes("expose") ||
+      lower.includes("allow")
+    ) {
+      type = "feature";
+    } else if (lower.includes("breaking")) {
+      type = "breaking";
+    }
+
+    entries.push({ text, type });
+  }
+
+  return entries;
+}
+
+// --- Git + GitHub helpers for finding PR numbers ---
+
+const REPO = "triggerdotdev/trigger.dev";
+
+function gitExec(args) {
+  return new Promise((resolve, reject) => {
+    execFile("git", args, { cwd: ROOT_DIR, maxBuffer: 1024 * 1024 }, (err, stdout) => {
+      if (err) reject(err);
+      else resolve(stdout.trim());
+    });
+  });
+}
+
+async function getCommitForFile(filePath) {
+  try {
+    // Find the commit that added this file
+    const sha = await gitExec(["log", "--diff-filter=A", "--format=%H", "--", filePath]);
+    return sha.split("\n")[0] || null;
+  } catch {
+    return null;
+  }
+}
+
+async function getPrForCommit(commitSha) {
+  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+  if (!token || !commitSha) return null;
+
+  try {
+    const res = await fetch(`https://api.github.com/repos/${REPO}/commits/${commitSha}/pulls`, {
+      headers: {
+        Authorization: `token ${token}`,
+        Accept: "application/vnd.github.v3+json",
+      },
+    });
+    if (!res.ok) return null;
+
+    const pulls = await res.json();
+    if (!pulls.length) return null;
+
+    // Prefer merged PRs, earliest merge first (same logic as @changesets/get-github-info)
+    const sorted = pulls.sort((a, b) => {
+      if (!a.merged_at && !b.merged_at) return 0;
+      if (!a.merged_at) return 1;
+      if (!b.merged_at) return -1;
+      return new Date(a.merged_at) - new Date(b.merged_at);
+    });
+
+    return sorted[0].number;
+  } catch {
+    return null;
+  }
+}
+
+// --- Parse .server-changes/ files ---
+
+async function parseServerChanges() {
+  const entries = [];
+  const fileData = await getServerChangeFileData();
+
+  // Look up commits for all files in parallel
+  const commits = await Promise.all(fileData.map((f) => getCommitForFile(f.filePath)));
+
+  // Look up PRs for all commits in parallel
+  const prNumbers = await Promise.all(commits.map((sha) => getPrForCommit(sha)));
+
+  for (let i = 0; i < fileData.length; i++) {
+    const { parsed } = fileData[i];
+    let text = parsed.body.trim();
+    const pr = prNumbers[i];
+
+    // Append PR link if we found one and it's not already in the text
+    if (pr && !text.includes(`#${pr}`)) {
+      text += ` ([#${pr}](https://github.com/${REPO}/pull/${pr}))`;
+    }
+
+    entries.push({
+      text,
+      type: parsed.frontmatter.type || "improvement",
+      area: parsed.frontmatter.area || "webapp",
+    });
+  }
+
+  return entries;
+}
+
+async function getServerChangeFileData() {
+  // The changesets version command deletes .server-changes before this script
+  // enhances the release PR body. We combine files still live on disk with the
+  // ones recovered from the release branch diff, deduped by filename, rather
+  // than picking one source or the other. This is additive so a partial cleanup
+  // (some files deleted, some still live) can't silently drop entries. Live
+  // files win on collision since they are the current on-disk truth.
+  const [live, deleted] = await Promise.all([
+    getLiveServerChangeFileData(),
+    getDeletedServerChangeFileDataFromReleaseBranch(),
+  ]);
+
+  const byName = new Map();
+  for (const fileData of deleted) {
+    byName.set(fileData.filePath.split("/").pop(), fileData);
+  }
+  for (const fileData of live) {
+    byName.set(fileData.filePath.split("/").pop(), fileData);
+  }
+
+  return [...byName.values()].sort((a, b) => a.filePath.localeCompare(b.filePath));
+}
+
+async function getLiveServerChangeFileData() {
+  const dir = join(ROOT_DIR, ".server-changes");
+
+  let files;
+  try {
+    files = await fs.readdir(dir);
+  } catch {
+    return [];
+  }
+
+  const fileData = [];
+  for (const file of files.sort()) {
+    if (!file.endsWith(".md") || file === "README.md") continue;
+
+    const filePath = join(".server-changes", file);
+    const content = await fs.readFile(join(dir, file), "utf-8");
+    const parsed = parseFrontmatter(content);
+    if (!parsed.body.trim()) continue;
+
+    fileData.push({ filePath, parsed });
+  }
+
+  return fileData;
+}
+
+async function getDeletedServerChangeFileDataFromReleaseBranch() {
+  const baseRef = process.env.SERVER_CHANGES_BASE_REF || "origin/main";
+  const releaseRef = process.env.SERVER_CHANGES_RELEASE_REF || "origin/changeset-release/main";
+
+  let mergeBase;
+  let deletedFiles;
+  try {
+    mergeBase = await gitExec(["merge-base", baseRef, releaseRef]);
+    deletedFiles = await gitExec([
+      "diff",
+      "--name-only",
+      "--diff-filter=D",
+      `${mergeBase}..${releaseRef}`,
+      "--",
+      ".server-changes",
+    ]);
+  } catch (err) {
+    console.error(
+      "[enhance-release-pr] failed to recover deleted server-changes from release branch:",
+      err
+    );
+    return [];
+  }
+
+  const fileData = [];
+  for (const filePath of deletedFiles.split("\n").filter(Boolean).sort()) {
+    const file = filePath.split("/").pop();
+    if (!file?.endsWith(".md") || file === "README.md") continue;
+
+    try {
+      const content = await gitExec(["show", `${mergeBase}:${filePath}`]);
+      const parsed = parseFrontmatter(content);
+      if (!parsed.body.trim()) continue;
+      fileData.push({ filePath, parsed });
+    } catch (err) {
+      // If an individual file cannot be recovered, skip it rather than hiding
+      // all other server changes.
+      console.error(`[enhance-release-pr] failed to read deleted server-change ${filePath}:`, err);
+    }
+  }
+
+  return fileData;
+}
+
+function parseFrontmatter(content) {
+  const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!match) return { frontmatter: {}, body: content };
+
+  const frontmatter = {};
+  for (const line of match[1].split("\n")) {
+    const [key, ...rest] = line.split(":");
+    if (key && rest.length) {
+      frontmatter[key.trim()] = rest.join(":").trim();
+    }
+  }
+
+  return { frontmatter, body: match[2] };
+}
+
+// --- Format the enhanced PR body ---
+
+// Render an entry as a list item, re-indenting continuation lines (code blocks,
+// sub-bullets, paragraphs) by 2 spaces so they stay inside the "- " bullet.
+function renderEntry(text) {
+  return `- ${text.replace(/\n/g, "\n  ")}`;
+}
+
+function formatPrBody({ version, packageEntries, serverEntries, rawBody }) {
+  const lines = [];
+
+  const features = packageEntries.filter((e) => e.type === "feature");
+  const fixes = packageEntries.filter((e) => e.type === "fix");
+  const improvements = packageEntries.filter((e) => e.type === "improvement" || e.type === "other");
+  const breaking = packageEntries.filter((e) => e.type === "breaking");
+
+  const serverFeatures = serverEntries.filter((e) => e.type === "feature");
+  const serverFixes = serverEntries.filter((e) => e.type === "fix");
+  const serverImprovements = serverEntries.filter((e) => e.type === "improvement");
+  const serverBreaking = serverEntries.filter((e) => e.type === "breaking");
+
+  const totalFeatures = features.length + serverFeatures.length;
+  const totalFixes = fixes.length + serverFixes.length;
+  const totalImprovements = improvements.length + serverImprovements.length;
+
+  // Summary line
+  const parts = [];
+  if (totalFeatures > 0) parts.push(`${totalFeatures} new feature${totalFeatures > 1 ? "s" : ""}`);
+  if (totalImprovements > 0)
+    parts.push(`${totalImprovements} improvement${totalImprovements > 1 ? "s" : ""}`);
+  if (totalFixes > 0) parts.push(`${totalFixes} bug fix${totalFixes > 1 ? "es" : ""}`);
+  if (parts.length > 0) {
+    lines.push(`## Summary`);
+    lines.push(`${parts.join(", ")}.`);
+    lines.push("");
+  }
+
+  // Breaking changes
+  if (breaking.length > 0 || serverBreaking.length > 0) {
+    lines.push("## Breaking changes");
+    for (const entry of [...breaking, ...serverBreaking]) lines.push(renderEntry(entry.text));
+    lines.push("");
+  }
+
+  // Highlights (features)
+  if (features.length > 0) {
+    lines.push("## Highlights");
+    lines.push("");
+    for (const entry of features) {
+      lines.push(renderEntry(entry.text));
+    }
+    lines.push("");
+  }
+
+  // Improvements
+  if (improvements.length > 0) {
+    lines.push("## Improvements");
+    for (const entry of improvements) lines.push(renderEntry(entry.text));
+    lines.push("");
+  }
+
+  // Bug fixes
+  if (fixes.length > 0) {
+    lines.push("## Bug fixes");
+    for (const entry of fixes) lines.push(renderEntry(entry.text));
+    lines.push("");
+  }
+
+  // Server changes
+  const allServer = [...serverFeatures, ...serverImprovements, ...serverFixes];
+  if (allServer.length > 0) {
+    lines.push("## Server changes");
+    lines.push("");
+    lines.push("These changes affect the self-hosted Docker image and Trigger.dev Cloud:");
+    lines.push("");
+    for (const entry of allServer) {
+      lines.push(renderEntry(entry.text));
+    }
+    lines.push("");
+  }
+
+  // Raw changeset output in collapsed section
+  if (rawBody) {
+    // Strip the Changesets action boilerplate from the raw body
+    const cleanedBody = rawBody
+      .replace(
+        /This PR was opened by the \[Changesets release\].*?If you're not ready to do a release yet.*?\n/gs,
+        ""
+      )
+      .trim();
+
+    if (cleanedBody) {
+      lines.push("<details>");
+      lines.push("<summary>Raw changeset output</summary>");
+      lines.push("");
+      lines.push(cleanedBody);
+      lines.push("");
+      lines.push("</details>");
+    }
+  }
+
+  return lines.join("\n");
+}
+
+// --- Main ---
+
+async function main() {
+  let rawBody = process.env.CHANGESET_PR_BODY || "";
+  if (!rawBody && !process.stdin.isTTY) {
+    const chunks = [];
+    for await (const chunk of process.stdin) chunks.push(chunk);
+    rawBody = Buffer.concat(chunks).toString("utf-8");
+  }
+
+  const packageEntries = parsePrBody(rawBody);
+  const serverEntries = await parseServerChanges();
+
+  const body = formatPrBody({
+    version,
+    packageEntries,
+    serverEntries,
+    rawBody,
+  });
+
+  process.stdout.write(body);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

scripts/generate-github-release.mjs

@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+
+/**
+ * Generates a unified GitHub release body for a trigger.dev version release.
+ *
+ * Usage:
+ *   node scripts/generate-github-release.mjs <version>
+ *
+ * Reads:
+ *   - The enhanced changeset release PR body (via RELEASE_PR_BODY env var or stdin).
+ *     By the time this runs, the PR body has already been enhanced by enhance-release-pr.mjs
+ *     to include server changes, deduplication, and categorization. The .server-changes/ files
+ *     themselves are already deleted (consumed on the release branch, same as .changeset/ files).
+ *   - Git log for contributor info
+ *
+ * Outputs the formatted GitHub release body to stdout.
+ */
+
+import { execSync } from "child_process";
+import { readdirSync, readFileSync } from "fs";
+import { join } from "path";
+
+const version = process.argv[2];
+if (!version) {
+  console.error("Usage: node scripts/generate-github-release.mjs <version>");
+  process.exit(1);
+}
+
+const ROOT_DIR = join(import.meta.dirname, "..");
+
+// --- Parse the enhanced PR body ---
+// The PR body from enhance-release-pr.mjs has sections like:
+//   ## Highlights
+//   ## Improvements
+//   ## Bug fixes
+//   ## Server changes
+//   ## Breaking changes
+//   <details>...</details>
+// We extract the content between the first heading and the <details> block.
+
+function extractChangesFromPrBody(body) {
+  if (!body) return "";
+
+  const lines = body.split("\n");
+  const outputLines = [];
+  let inDetails = false;
+  let inSummary = false;
+  let foundContent = false;
+
+  for (const line of lines) {
+    // Skip the title line (# trigger.dev vX.Y.Z)
+    if (line.startsWith("# trigger.dev v")) continue;
+
+    // Skip the entire Summary section (heading + content until next heading)
+    if (line.startsWith("## Summary")) {
+      inSummary = true;
+      continue;
+    }
+    if (inSummary) {
+      if (line.startsWith("## ")) {
+        inSummary = false;
+      } else {
+        continue;
+      }
+    }
+
+    // Stop before raw changeset output
+    if (line.trim() === "<details>") {
+      inDetails = true;
+      continue;
+    }
+    if (inDetails) continue;
+
+    // Collect everything from the first ## heading onward
+    if (line.startsWith("## ") && !foundContent) {
+      foundContent = true;
+    }
+
+    if (foundContent) {
+      outputLines.push(line);
+    }
+  }
+
+  return outputLines.join("\n").trim();
+}
+
+// --- Get contributors from git log ---
+
+function getContributors(previousVersion) {
+  try {
+    const range = previousVersion ? `v${previousVersion}...HEAD` : "HEAD~50..HEAD";
+    const log = execSync(`git log ${range} --format="%aN|%aE" --no-merges`, {
+      cwd: ROOT_DIR,
+      encoding: "utf-8",
+    });
+
+    const contributors = new Map();
+    for (const line of log.split("\n").filter(Boolean)) {
+      const [name, email] = line.split("|");
+      if (!name || email?.endsWith("@users.noreply.github.com")) {
+        // Try to extract username from noreply email
+        const match = email?.match(/(\d+\+)?(.+)@users\.noreply\.github\.com/);
+        if (match) {
+          const username = match[2];
+          contributors.set(username, (contributors.get(username) || 0) + 1);
+        }
+        continue;
+      }
+      contributors.set(name, (contributors.get(name) || 0) + 1);
+    }
+
+    return [...contributors.entries()].sort((a, b) => b[1] - a[1]).map(([name]) => name);
+  } catch {
+    return [];
+  }
+}
+
+// --- Get published packages ---
+
+function getPublishedPackages() {
+  try {
+    const packagesDir = join(ROOT_DIR, "packages");
+    const names = [];
+    for (const dir of readdirSync(packagesDir, { withFileTypes: true })) {
+      if (!dir.isDirectory()) continue;
+      try {
+        const pkg = JSON.parse(readFileSync(join(packagesDir, dir.name, "package.json"), "utf-8"));
+        if (pkg.name && !pkg.private) {
+          names.push(pkg.name);
+        }
+      } catch {
+        // skip directories without package.json
+      }
+    }
+    return names.sort();
+  } catch {
+    return [
+      "@trigger.dev/build",
+      "@trigger.dev/core",
+      "@trigger.dev/react-hooks",
+      "@trigger.dev/sdk",
+      "trigger.dev",
+    ];
+  }
+}
+
+function getPreviousVersion(version) {
+  const parts = version.split(".").map(Number);
+  if (parts[2] > 0) {
+    parts[2]--;
+  } else if (parts[1] > 0) {
+    parts[1]--;
+    parts[2] = 0;
+  } else if (parts[0] > 0) {
+    parts[0]--;
+    parts[1] = 0;
+    parts[2] = 0;
+  } else {
+    return null;
+  }
+  return parts.join(".");
+}
+
+// --- Format the release body ---
+
+function formatRelease({ version, changesContent, contributors, packages }) {
+  const lines = [];
+
+  lines.push(`# trigger.dev v${version}`);
+  lines.push("");
+  lines.push("## Upgrade");
+  lines.push("");
+  lines.push("```sh");
+  lines.push("npx trigger.dev@latest update  # npm");
+  lines.push("pnpm dlx trigger.dev@latest update  # pnpm");
+  lines.push("yarn dlx trigger.dev@latest update  # yarn");
+  lines.push("bunx trigger.dev@latest update  # bun");
+  lines.push("```");
+  lines.push("");
+  // The Docker image link initially points to the container page without a tag filter.
+  // After Docker images are built, the update-release job patches this with the exact tag URL.
+  lines.push(
+    `Self-hosted Docker image: [\`ghcr.io/triggerdotdev/trigger.dev:v${version}\`](https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev)`
+  );
+  lines.push("");
+  lines.push("## Release notes");
+  lines.push("");
+  lines.push(
+    `Read the full release notes: https://trigger.dev/changelog/v${version.replace(/\./g, "-")}`
+  );
+  lines.push("");
+
+  // What's changed — extracted from the enhanced PR body
+  if (changesContent) {
+    lines.push("## What's changed");
+    lines.push("");
+    lines.push(changesContent);
+    lines.push("");
+  }
+
+  // Packages
+  if (packages.length > 0) {
+    lines.push(`## All packages: v${version}`);
+    lines.push("");
+    lines.push(packages.join(", "));
+    lines.push("");
+  }
+
+  // Contributors
+  if (contributors.length > 0) {
+    lines.push("## Contributors");
+    lines.push("");
+    lines.push(
+      contributors.map((c) => (/^[A-Za-z0-9][-A-Za-z0-9]*$/.test(c) ? `@${c}` : c)).join(", ")
+    );
+    lines.push("");
+  }
+
+  // Comparison link
+  const prevVersion = getPreviousVersion(version);
+  if (prevVersion) {
+    lines.push(
+      `**Full changelog**: https://github.com/triggerdotdev/trigger.dev/compare/v${prevVersion}...v${version}`
+    );
+  }
+
+  return lines.join("\n");
+}
+
+// --- Main ---
+
+async function main() {
+  // Read PR body from env or stdin
+  let prBody = process.env.RELEASE_PR_BODY || "";
+  if (!prBody && !process.stdin.isTTY) {
+    const chunks = [];
+    for await (const chunk of process.stdin) chunks.push(chunk);
+    prBody = Buffer.concat(chunks).toString("utf-8");
+  }
+
+  const changesContent = extractChangesFromPrBody(prBody);
+
+  // Fail loudly when a stable (non-prerelease) release resolved no changelog. This
+  // guards against publishing an empty "What's changed" section, which previously
+  // happened silently on workflow_dispatch re-runs where RELEASE_PR_BODY was empty.
+  // Prereleases (any version with a hyphen, e.g. 4.5.0-rc.0) are allowed to be empty.
+  const isPrerelease = version.includes("-");
+  if (!changesContent && !isPrerelease) {
+    console.error(
+      `Error: no "What's changed" content could be resolved for v${version}. ` +
+        `RELEASE_PR_BODY was empty or contained no changelog entries. ` +
+        `Refusing to publish a release with an empty changelog.`
+    );
+    process.exit(1);
+  }
+
+  const contributors = getContributors(getPreviousVersion(version));
+  const packages = getPublishedPackages();
+
+  const body = formatRelease({
+    version,
+    changesContent,
+    contributors,
+    packages,
+  });
+
+  process.stdout.write(body);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

scripts/publish-prerelease.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -e
+
+# Use the first argument as version or 'v3-prerelease' if not available
+version=${1:-'v4-prerelease'}
+
+# Ensure git stage is clear
+if [[ $(git status --porcelain) ]]; then
+    echo "Your git status is not clean. Please commit your changes before running this script.";
+    echo "To reset all your changes, run this instead: git reset --hard HEAD"
+    exit 1;
+else
+    echo "Git status is clean. Proceeding with the script.";
+fi
+
+# From here on, if the user aborts the script, we will clean up the git stage
+git_reset() {
+    git reset --hard HEAD
+}
+abort() {
+    echo "Aborted. Cleaning up..."
+    git_reset
+    exit 1
+}
+trap abort INT
+
+# Run your commands
+# Run changeset version command and capture its output
+echo "Running: pnpm exec changeset version --snapshot $version"
+if output=$(pnpm exec changeset version --snapshot $version 2>&1); then
+    if echo "$output" | grep -q "No unreleased changesets found"; then
+        echo "No unreleased changesets found. Exiting."
+        exit 0
+    fi
+else
+    echo "$output"
+    echo "Error running changeset version command, detailed output above"
+    exit 1
+fi
+
+read -e -p "Pausing for manual changes, press Enter when ready to continue..."
+
+echo "Running: pnpm run clean --filter \"@trigger.dev/*\" --filter \"trigger.dev\""
+pnpm run clean --filter "@trigger.dev/*" --filter "trigger.dev"
+
+echo "Running: pnpm run build --filter \"@trigger.dev/*\" --filter \"trigger.dev\""
+pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+echo "Going to run: pnpm exec changeset publish --no-git-tag --snapshot --tag $version"
+read -p "Do you wish to continue? (y/N): " prompt
+if [[ $prompt =~ [yY](es)* ]]; then
+    pnpm exec changeset publish --no-git-tag --snapshot --tag $version
+else
+    abort
+fi
+
+# If there were no errors, clear the git stage
+echo "Commands ran successfully. Clearing the git stage."
+git_reset

scripts/recover-stuck-runs.ts

@@ -0,0 +1,374 @@
+#!/usr/bin/env tsx
+
+/**
+ * Recovery script for runs stuck in currentConcurrency with QUEUED execution status
+ *
+ * PROBLEM:
+ * During high database load, runs can get dequeued from Redis (added to currentConcurrency)
+ * but fail to update their execution status in the database. This leaves them stuck in an
+ * inconsistent state where they won't be re-dequeued because they're marked as "in progress"
+ * in Redis, but their database state still shows QUEUED.
+ *
+ * SOLUTION:
+ * This script identifies and recovers these stuck runs by:
+ * 1. Reading from the environment currentConcurrency Redis set
+ * 2. Checking which runs have QUEUED execution status (inconsistent state)
+ * 3. Re-adding them to their specific queue sorted sets
+ * 4. Removing them from the queue-specific currentConcurrency sets
+ * 5. Removing them from the environment-level currentConcurrency set
+ *
+ * SAFETY:
+ * - Dry-run mode when no write Redis URL is provided (read-only, no writes)
+ * - Uses separate Redis connections for reads and writes
+ * - Write connection only created when redisWriteUrl is provided
+ *
+ * ARGUMENTS:
+ *   <environmentId>   The Trigger.dev environment ID (e.g., env_abc123)
+ *   <postgresUrl>     PostgreSQL connection string
+ *   <redisReadUrl>    Redis connection string for reads (redis:// or rediss://)
+ *   [redisWriteUrl]   Optional Redis connection string for writes (omit for dry-run)
+ *
+ * USAGE:
+ *   tsx scripts/recover-stuck-runs.ts <environmentId> <postgresUrl> <redisReadUrl> [redisWriteUrl]
+ *
+ * EXAMPLES:
+ *
+ *   Dry-run mode (safe, no writes):
+ *   tsx scripts/recover-stuck-runs.ts env_1234567890 \
+ *     "postgresql://user:pass@localhost:5432/triggerdev" \
+ *     "redis://readonly.example.com:6379"
+ *
+ *   Execute mode (makes actual changes):
+ *   tsx scripts/recover-stuck-runs.ts env_1234567890 \
+ *     "postgresql://user:pass@localhost:5432/triggerdev" \
+ *     "redis://readonly.example.com:6379" \
+ *     "redis://writeonly.example.com:6379"
+ */
+
+import type { TaskRunExecutionStatus } from "@trigger.dev/database";
+import { PrismaClient } from "@trigger.dev/database";
+import { createRedisClient } from "@internal/redis";
+
+interface StuckRun {
+  runId: string;
+  orgId: string;
+  projectId: string;
+  environmentId: string;
+  queue: string;
+  concurrencyKey: string | null;
+  executionStatus: TaskRunExecutionStatus;
+  snapshotCreatedAt: Date;
+  taskIdentifier: string;
+}
+
+interface RedisOperation {
+  type: "ZADD" | "SREM";
+  key: string;
+  args: (string | number)[];
+  description: string;
+}
+
+async function main() {
+  const [environmentId, postgresUrl, redisReadUrl, redisWriteUrl] = process.argv.slice(2);
+
+  if (!environmentId || !postgresUrl || !redisReadUrl) {
+    console.error(
+      "Usage: tsx scripts/recover-stuck-runs.ts <environmentId> <postgresUrl> <redisReadUrl> [redisWriteUrl]"
+    );
+    console.error("");
+    console.error("Dry-run mode when no redisWriteUrl is provided (read-only).");
+    console.error("Execute mode when redisWriteUrl is provided (makes actual changes).");
+    console.error("");
+    console.error("Example (dry-run):");
+    console.error("  tsx scripts/recover-stuck-runs.ts env_1234567890 \\");
+    console.error('    "postgresql://user:pass@localhost:5432/triggerdev" \\');
+    console.error('    "redis://readonly.example.com:6379"');
+    console.error("");
+    console.error("Example (execute):");
+    console.error("  tsx scripts/recover-stuck-runs.ts env_1234567890 \\");
+    console.error('    "postgresql://user:pass@localhost:5432/triggerdev" \\');
+    console.error('    "redis://readonly.example.com:6379" \\');
+    console.error('    "redis://writeonly.example.com:6379"');
+    process.exit(1);
+  }
+
+  const executeMode = !!redisWriteUrl;
+
+  if (executeMode) {
+    console.log("⚠️  EXECUTE MODE - Changes will be made to Redis\n");
+  } else {
+    console.log("🔍 DRY RUN MODE - No changes will be made to Redis\n");
+  }
+
+  console.log(`🔍 Scanning for stuck runs in environment: ${environmentId}`);
+
+  // Create Prisma client with the provided connection URL
+  const prisma = new PrismaClient({
+    datasources: {
+      db: {
+        url: postgresUrl,
+      },
+    },
+  });
+
+  try {
+    // Get environment details
+    const environment = await prisma.runtimeEnvironment.findUnique({
+      where: { id: environmentId },
+      include: {
+        organization: true,
+        project: true,
+      },
+    });
+
+    if (!environment) {
+      console.error(`❌ Environment not found: ${environmentId}`);
+      process.exit(1);
+    }
+
+    console.log(`📍 Environment: ${environment.slug} (${environment.type})`);
+    console.log(`📍 Organization: ${environment.organization.slug}`);
+    console.log(`📍 Project: ${environment.project.slug}`);
+
+    // Parse Redis read URL
+    const redisReadUrlObj = new URL(redisReadUrl);
+    const redisReadOptions = {
+      host: redisReadUrlObj.hostname,
+      port: parseInt(redisReadUrlObj.port || "6379"),
+      username: redisReadUrlObj.username || undefined,
+      password: redisReadUrlObj.password || undefined,
+      enableAutoPipelining: false,
+      ...(redisReadUrlObj.protocol === "rediss:"
+        ? {
+            tls: {
+              // If connecting via localhost tunnel to a remote Redis, disable cert verification
+              rejectUnauthorized: redisReadUrlObj.hostname === "localhost" ? false : true,
+            },
+          }
+        : {}),
+    };
+
+    // Create Redis read client
+    const redisRead = createRedisClient(redisReadOptions);
+
+    // Create Redis write client if redisWriteUrl is provided
+    let redisWrite = null;
+    if (redisWriteUrl) {
+      const redisWriteUrlObj = new URL(redisWriteUrl);
+      const redisWriteOptions = {
+        host: redisWriteUrlObj.hostname,
+        port: parseInt(redisWriteUrlObj.port || "6379"),
+        username: redisWriteUrlObj.username || undefined,
+        password: redisWriteUrlObj.password || undefined,
+        enableAutoPipelining: false,
+        ...(redisWriteUrlObj.protocol === "rediss:"
+          ? {
+              tls: {
+                // If connecting via localhost tunnel to a remote Redis, disable cert verification
+                rejectUnauthorized: redisWriteUrlObj.hostname === "localhost" ? false : true,
+              },
+            }
+          : {}),
+      };
+      redisWrite = createRedisClient(redisWriteOptions);
+    }
+
+    try {
+      // Build the Redis key for environment-level currentConcurrency set
+      // Format: engine:runqueue:{org:X}:proj:Y:env:Z:currentConcurrency
+      const envConcurrencyKey = `engine:runqueue:{org:${environment.organizationId}}:proj:${environment.projectId}:env:${environmentId}:currentConcurrency`;
+
+      console.log(`\n🔑 Checking Redis key: ${envConcurrencyKey}`);
+
+      // Get all run IDs in the environment's currentConcurrency set
+      const runIds = await redisRead.smembers(envConcurrencyKey);
+
+      if (runIds.length === 0) {
+        console.log(`✅ No runs in currentConcurrency set`);
+        return;
+      }
+
+      console.log(`📊 Found ${runIds.length} runs in currentConcurrency set`);
+
+      // Query database for latest snapshots and queue info of these runs.
+      // NOTE: raw join of TaskRunExecutionSnapshot to TaskRun, the one TaskRun read not behind
+      // RunStore (a join, not a by-id read, in an ops script). Revisit at table cutover.
+      const runInfo = await prisma.$queryRaw<
+        Array<{
+          runId: string;
+          executionStatus: TaskRunExecutionStatus;
+          snapshotCreatedAt: Date;
+          organizationId: string;
+          projectId: string;
+          environmentId: string;
+          taskIdentifier: string;
+          queue: string;
+          concurrencyKey: string | null;
+        }>
+      >`
+        SELECT DISTINCT ON (s."runId")
+          s."runId",
+          s."executionStatus",
+          s."createdAt" as "snapshotCreatedAt",
+          r."organizationId",
+          r."projectId",
+          r."runtimeEnvironmentId" as "environmentId",
+          r."taskIdentifier",
+          r."queue",
+          r."concurrencyKey"
+        FROM "TaskRunExecutionSnapshot" s
+        INNER JOIN "TaskRun" r ON r.id = s."runId"
+        WHERE s."runId" = ANY(${runIds})
+          AND s."isValid" = true
+        ORDER BY s."runId", s."createdAt" DESC
+      `;
+
+      const stuckRuns: StuckRun[] = [];
+
+      // Find runs with QUEUED execution status (inconsistent state)
+      for (const info of runInfo) {
+        if (info.executionStatus === "QUEUED") {
+          stuckRuns.push({
+            runId: info.runId,
+            orgId: info.organizationId,
+            projectId: info.projectId,
+            environmentId: info.environmentId,
+            queue: info.queue,
+            concurrencyKey: info.concurrencyKey,
+            executionStatus: info.executionStatus,
+            snapshotCreatedAt: info.snapshotCreatedAt,
+            taskIdentifier: info.taskIdentifier,
+          });
+        }
+      }
+
+      if (stuckRuns.length === 0) {
+        console.log(`✅ No stuck runs found (all runs have progressed beyond QUEUED state)`);
+        return;
+      }
+
+      console.log(`\n⚠️  Found ${stuckRuns.length} stuck runs in QUEUED state:`);
+      console.log(`════════════════════════════════════════════════════════════════`);
+
+      for (const run of stuckRuns) {
+        const age = Date.now() - run.snapshotCreatedAt.getTime();
+        const ageMinutes = Math.floor(age / 1000 / 60);
+        console.log(`  • Run: ${run.runId}`);
+        console.log(`    Task: ${run.taskIdentifier}`);
+        console.log(`    Queue: ${run.queue}`);
+        console.log(`    Concurrency Key: ${run.concurrencyKey || "(none)"}`);
+        console.log(`    Status: ${run.executionStatus}`);
+        console.log(`    Stuck for: ${ageMinutes} minutes`);
+        console.log(`    Snapshot created: ${run.snapshotCreatedAt.toISOString()}`);
+        console.log();
+      }
+
+      // Prepare recovery operations
+      console.log(
+        `\n⚡ ${executeMode ? "Executing" : "Planning"} recovery for ${stuckRuns.length} stuck runs`
+      );
+      console.log(`This will:`);
+      console.log(`  1. Add each run back to its specific queue sorted set`);
+      console.log(`  2. Remove each run from the queue-specific currentConcurrency set`);
+      console.log(`  3. Remove each run from the env-level currentConcurrency set`);
+      console.log();
+
+      let successCount = 0;
+      let failureCount = 0;
+
+      const currentTimestamp = Date.now();
+
+      for (const run of stuckRuns) {
+        try {
+          // Build queue key: engine:runqueue:{org:X}:proj:Y:env:Z:queue:QUEUENAME
+          // Build queue currentConcurrency key: engine:runqueue:{org:X}:proj:Y:env:Z:queue:QUEUENAME:currentConcurrency
+          const queueKey = run.concurrencyKey
+            ? `engine:runqueue:{org:${run.orgId}}:proj:${run.projectId}:env:${run.environmentId}:queue:${run.queue}:ck:${run.concurrencyKey}`
+            : `engine:runqueue:{org:${run.orgId}}:proj:${run.projectId}:env:${run.environmentId}:queue:${run.queue}`;
+
+          const queueConcurrencyKey = `${queueKey}:currentConcurrency`;
+
+          const operations: RedisOperation[] = [
+            {
+              type: "ZADD",
+              key: queueKey,
+              args: [currentTimestamp, run.runId],
+              description: `Add run to queue sorted set with score ${currentTimestamp}`,
+            },
+            {
+              type: "SREM",
+              key: queueConcurrencyKey,
+              args: [run.runId],
+              description: `Remove run from queue currentConcurrency set`,
+            },
+            {
+              type: "SREM",
+              key: envConcurrencyKey,
+              args: [run.runId],
+              description: `Remove run from env currentConcurrency set`,
+            },
+          ];
+
+          if (executeMode && redisWrite) {
+            // Execute operations using the write client
+            await redisWrite.zadd(queueKey, currentTimestamp, run.runId);
+            const removedFromQueue = await redisWrite.srem(queueConcurrencyKey, run.runId);
+            const removedFromEnv = await redisWrite.srem(envConcurrencyKey, run.runId);
+
+            console.log(`  ✓ Recovered run ${run.runId} (${run.taskIdentifier})`);
+            if (removedFromQueue === 0) {
+              console.log(`    ⚠ Run was not in queue currentConcurrency set`);
+            }
+            if (removedFromEnv === 0) {
+              console.log(`    ⚠ Run was not in env currentConcurrency set`);
+            }
+            successCount++;
+          } else {
+            // Dry run - just show what would be done
+            console.log(`  📝 Would recover run ${run.runId} (${run.taskIdentifier}):`);
+            for (const op of operations) {
+              console.log(`     ${op.type} ${op.key}`);
+              console.log(`       Args: ${JSON.stringify(op.args)}`);
+              console.log(`       (${op.description})`);
+            }
+            successCount++;
+          }
+        } catch (error) {
+          console.error(`  ✗ Failed to recover run ${run.runId}:`, error);
+          failureCount++;
+        }
+      }
+
+      console.log(`\n═══════════════════════════════════════════════════════════════`);
+      if (executeMode) {
+        console.log(`✅ Recovery complete!`);
+        console.log(`   Recovered: ${successCount}`);
+        console.log(`   Failed: ${failureCount}`);
+        console.log();
+        console.log(`ℹ️  Note: The recovered runs should be automatically dequeued`);
+        console.log(`   by the master queue consumers within a few seconds.`);
+      } else {
+        console.log(`📋 Dry run complete - no changes were made`);
+        console.log(`   Would recover: ${successCount}`);
+        console.log(`   Would fail: ${failureCount}`);
+        console.log();
+        console.log(`💡 To execute these changes, run again with a redisWriteUrl argument`);
+      }
+    } finally {
+      await redisRead.quit();
+      if (redisWrite) {
+        await redisWrite.quit();
+      }
+    }
+  } catch (error) {
+    console.error("❌ Error during recovery:", error);
+    throw error;
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+
+main().catch((error) => {
+  console.error("Fatal error:", error);
+  process.exit(1);
+});

scripts/retry-prisma-generate.mjs

@@ -0,0 +1,53 @@
+// Retry wrapper around `prisma generate`. Our two prisma clients pin the same
+// prisma version and so share one package instance in the pnpm store; when
+// `turbo run generate` runs them concurrently both race to write the shared
+// query-engine binary, and on Windows the loser fails with `EPERM ... rename`.
+// Retrying lets it succeed once the engine file is present and unlocked. On
+// non-Windows the first attempt succeeds, so this is a zero-cost no-op.
+import { spawnSync } from "node:child_process";
+
+const MAX_ATTEMPTS = 5;
+const BASE_DELAY_MS = 500;
+
+// Transient, retryable filesystem contention on the shared engine binary.
+const TRANSIENT =
+  /\b(EPERM|EBUSY|EACCES)\b|operation not permitted|resource busy or locked|being used by another process/i;
+
+const passthroughArgs = process.argv.slice(2);
+
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+let lastStatus = 1;
+
+for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+  const result = spawnSync("prisma", ["generate", ...passthroughArgs], {
+    shell: true,
+    encoding: "utf8",
+  });
+
+  process.stdout.write(result.stdout ?? "");
+  process.stderr.write(result.stderr ?? "");
+
+  if (result.status === 0) {
+    process.exit(0);
+  }
+
+  lastStatus = result.status ?? 1;
+
+  const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
+  const isRetryable = TRANSIENT.test(output);
+
+  if (!isRetryable || attempt === MAX_ATTEMPTS) {
+    break;
+  }
+
+  const delay = BASE_DELAY_MS * attempt;
+  console.error(
+    `prisma generate hit a transient filesystem error (attempt ${attempt}/${MAX_ATTEMPTS}); retrying in ${delay}ms...`
+  );
+  sleepSync(delay);
+}
+
+process.exit(lastStatus);

scripts/stamp-preview-version.mjs

@@ -0,0 +1,90 @@
+// Rewrites the version of every public packages/* package to a unique,
+// preview-only semver (0.0.0-preview-<sha>) BEFORE the build runs.
+//
+// Why this exists:
+//   pkg.pr.new serves preview builds by commit SHA but does NOT change the
+//   package.json "version" field (as of 0.0.75). If a preview is published
+//   while package.json still says e.g. 4.5.0-rc.4, a consumer who installs the
+//   preview pins 4.5.0-rc.4 to the pkg.pr.new tarball in their lockfile/cache,
+//   and a later `npm i @trigger.dev/sdk@4.5.0-rc.4` from npm can resolve to the
+//   stale preview. See stackblitz-labs/pkg.pr.new#250 and #390.
+//
+//   A 0.0.0- prefix can never satisfy a real semver range, so the collision
+//   becomes structurally impossible (the same convention React/Next canaries
+//   use).
+//
+//   Running BEFORE the build also means scripts/updateVersion.ts bakes this
+//   same preview version into the runtime VERSION constant, so previews are
+//   self-identifying (trigger --version, the x-trigger-cli-version header, the
+//   MCP server version, etc.) rather than all reporting the RC version.
+//
+//   Sibling workspace: specifiers are relaxed to workspace:* so `pnpm pack`
+//   resolves them against the rewritten versions without range-validation
+//   errors. packages/python pins peerDependencies as workspace:^4.5.0-rc.4,
+//   which would otherwise be unsatisfiable once the sibling version changes.
+//
+// Note: pkg.pr.new PR #525 adds a built-in --previewVersion flag. Once that
+// ships we could drop the version-rewrite half here, but we still want a
+// pre-build stamp so updateVersion.ts picks up the preview version (the flag
+// rewrites at pack time, which is too late for the baked VERSION constant).
+
+import { readdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { execSync } from "node:child_process";
+
+const PACKAGES_DIR = "packages";
+const DEP_SECTIONS = [
+  "dependencies",
+  "devDependencies",
+  "peerDependencies",
+  "optionalDependencies",
+];
+
+function resolveSha() {
+  const sha = process.argv[2] || process.env.GITHUB_SHA;
+  if (sha) return sha;
+  try {
+    return execSync("git rev-parse HEAD", { encoding: "utf8" }).trim();
+  } catch {
+    throw new Error(
+      "Could not determine commit SHA (pass as the first argument or set GITHUB_SHA)"
+    );
+  }
+}
+
+const sha = resolveSha().slice(0, 7);
+const previewVersion = `0.0.0-preview-${sha}`;
+
+const dirs = readdirSync(PACKAGES_DIR, { withFileTypes: true }).filter((e) => e.isDirectory());
+
+// First pass: collect every public package name so we know which workspace
+// specifiers point at a sibling whose version we are about to change.
+const publicNames = new Set();
+const manifests = [];
+for (const dir of dirs) {
+  const pkgPath = join(PACKAGES_DIR, dir.name, "package.json");
+  if (!existsSync(pkgPath)) continue;
+  const json = JSON.parse(readFileSync(pkgPath, "utf8"));
+  manifests.push({ pkgPath, json });
+  if (!json.private && json.name) publicNames.add(json.name);
+}
+
+// Second pass: stamp the version and relax sibling workspace specifiers.
+let stamped = 0;
+for (const { pkgPath, json } of manifests) {
+  if (json.private) continue;
+  json.version = previewVersion;
+  for (const section of DEP_SECTIONS) {
+    const deps = json[section];
+    if (!deps) continue;
+    for (const [name, spec] of Object.entries(deps)) {
+      if (publicNames.has(name) && String(spec).startsWith("workspace:")) {
+        deps[name] = "workspace:*";
+      }
+    }
+  }
+  writeFileSync(pkgPath, `${JSON.stringify(json, null, 2)}\n`);
+  stamped++;
+}
+
+console.log(`Stamped ${stamped} public package(s) to ${previewVersion}`);

scripts/start-prometheus.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+prometheus --config.file=./.configs/prometheus.yml --storage.tsdb.path=/tmp/prom-data

scripts/unpack-worker.js

@@ -0,0 +1,68 @@
+const fs = require("fs");
+const zlib = require("zlib");
+const path = require("path");
+
+// Get the file paths from command line arguments
+let [jsonFilePath, destDir] = process.argv.slice(2);
+
+if (!jsonFilePath || !destDir) {
+  console.error("Usage: node script.js <json-file-path> <destination-directory>");
+  process.exit(1);
+}
+
+// Function to decompress the content
+function decompressContent(base64Encoded) {
+  // Decode base64 string to buffer
+  const compressedData = Buffer.from(base64Encoded, "base64");
+
+  // Decompress the data
+  const decompressedData = zlib.inflateSync(compressedData);
+
+  // Convert buffer to string
+  return decompressedData.toString();
+}
+
+try {
+  // Read and parse the JSON file
+  const jsonContent = fs.readFileSync(jsonFilePath, "utf8");
+
+  const data = JSON.parse(jsonContent)[0];
+
+  console.log(data);
+
+  const id = data.id;
+
+  console.log(`Extracting files for: ${id} to ${destDir}`);
+
+  destDir = path.join(destDir, id);
+
+  console.log(`Extracting files to: ${destDir}`);
+
+  // Create the destination directory if it doesn't exist
+  fs.mkdirSync(destDir, { recursive: true });
+
+  // Process each item in the array
+  const sourceFiles = data.metadata.sourceFiles;
+
+  sourceFiles.forEach((file) => {
+    // Decompress the contents
+    const decompressedContent = decompressContent(file.contents);
+
+    // Combine destination directory with file path
+    const fullPath = path.join(destDir, file.filePath);
+
+    // Create directory structure if it doesn't exist
+    const dirPath = path.dirname(fullPath);
+    fs.mkdirSync(dirPath, { recursive: true });
+
+    // Write the decompressed content to the file
+    fs.writeFileSync(fullPath, decompressedContent);
+
+    console.log(`Created file: ${fullPath}`);
+  });
+
+  console.log(`\nAll files have been extracted to: ${destDir}`);
+} catch (error) {
+  console.error(error);
+  process.exit(1);
+}

scripts/update-e2e-package.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# Print the current working directory
+echo "Updating packages in: $(pwd)"
+
+corepack use pnpm@8.15.5
+rm -rf **/node_modules
+
+# Check if package-lock.json exists in the current directory
+if [ -f "package-lock.json" ]; then
+    echo "package-lock.json found. Running npm install..."
+    npm install
+    rm -rf **/node_modules
+else
+    echo "No package-lock.json found. Skipping npm install."
+fi
+corepack use yarn@4.2.2

scripts/updateVersion.ts

@@ -0,0 +1,39 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { readPackageJSON } from "pkg-types";
+
+// This script will update the VERSION constant in the build output, found at:
+// {cwd}/dist/esm/version.js
+// {cwd}/dist/commonjs/version.js
+//
+// It fetches the version by reading the package.json file in the root of the project.
+async function updateVersion() {
+  const localPackageJson = await readPackageJSON(process.cwd());
+
+  if (!localPackageJson.version) {
+    throw new Error("Failed to read version from package.json");
+  }
+
+  const versionFileESM = path.join(process.cwd(), "dist", "esm", "version.js");
+  await updatePlaceholderInFile(versionFileESM, localPackageJson.version);
+
+  const versionFileCJS = path.join(process.cwd(), "dist", "commonjs", "version.js");
+  await updatePlaceholderInFile(versionFileCJS, localPackageJson.version);
+
+  console.log(
+    `Updated packages/${path.basename(process.cwd())} version.js to ${localPackageJson.version}`
+  );
+}
+
+async function updatePlaceholderInFile(filePath: string, version: string) {
+  try {
+    const fileContents = await fs.readFile(filePath, "utf-8");
+    const updatedContents = fileContents.replace("0.0.0", version);
+    await fs.writeFile(filePath, updatedContents);
+  } catch (_e) {}
+}
+
+updateVersion().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});