elizaos--eliza/.github/workflows/deploy-eliza-provisioning-worker.yml

name: Deploy Eliza Provisioning Worker

# SSHes to the Hetzner provisioning host and deploys one immutable commit,
# installs the systemd unit shipped at packages/scripts/cloud/admin/eliza-provisioning-worker.service,
# and restarts the eliza-provisioning-worker daemon.
#
# First-time setup:
#   HCLOUD_TOKEN=... bun run packages/scripts/cloud/admin/bootstrap-provisioning-worker-host.mjs

on:
  push:
    branches: [develop, main]
    paths:
      - '.github/workflows/deploy-eliza-provisioning-worker.yml'
      - 'packages/scripts/cloud/admin/daemons/provisioning-worker.ts'
      - 'packages/scripts/cloud/admin/daemons/agent-router.ts'
      - 'packages/scripts/cloud/admin/eliza-provisioning-worker.service'
      - 'packages/scripts/cloud/admin/eliza-agent-router.service'
      - 'packages/scripts/cloud/admin/ensure-generated-keywords.sh'
      - 'packages/cloud/shared/src/lib/services/provisioning-jobs.ts'
      - 'packages/cloud/shared/src/lib/services/docker-sandbox-provider.ts'
      - 'packages/cloud/shared/src/lib/services/eliza-sandbox.ts'
      - 'packages/cloud/shared/src/db/repositories/agent-sandboxes.ts'
      - 'packages/cloud/shared/**'
      - 'packages/cloud/sdk/**'
      - 'packages/shared/**'
      - 'packages/core/**'
      - 'plugins/plugin-sql/**'
  workflow_dispatch:
    inputs:
      environment:
        description: 'Target environment (host + secrets resolved from this)'
        required: true
        default: 'staging'
        type: choice
        options:
          - staging
          - production
      deployment_sha:
        description: 'Optional exact repository commit for protected staging acceptance'
        required: false
        type: string

# Per-env concurrency: a develop push (staging) and a main push (production)
# can deploy in parallel without one cancelling the other.
concurrency:
  group: deploy-eliza-provisioning-worker-${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'staging') }}
  cancel-in-progress: false

# Default to least privilege. Override per-job where needed.
permissions:
  contents: read

jobs:
  determine-env:
    name: Determine environment
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    outputs:
      environment: ${{ steps.env.outputs.environment }}
      branch: ${{ steps.env.outputs.branch }}
      deployment_sha: ${{ steps.snapshot.outputs.deployment_sha }}
    steps:
      - id: env
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            env="${{ github.event.inputs.environment }}"
          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
            env="production"
          else
            env="staging"
          fi
          branch=$([ "$env" = "production" ] && echo "main" || echo "develop")
          echo "environment=$env" >> "$GITHUB_OUTPUT"
          echo "branch=$branch"   >> "$GITHUB_OUTPUT"
          echo "Resolved: environment=$env branch=$branch"
      - id: snapshot
        env:
          BRANCH: ${{ steps.env.outputs.branch }}
          PUSH_SHA: ${{ github.sha }}
          REQUESTED_SHA: ${{ github.event.inputs.deployment_sha }}
          TARGET_ENVIRONMENT: ${{ steps.env.outputs.environment }}
        run: |
          set -euo pipefail
          if [ "${{ github.event_name }}" = "push" ]; then
            deployment_sha="$PUSH_SHA"
          elif [ -n "$REQUESTED_SHA" ]; then
            [ "$TARGET_ENVIRONMENT" = "staging" ] || {
              echo "::error::An exact deployment_sha is permitted only for protected staging deployments"
              exit 1
            }
            [[ "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]] || {
              echo "::error::deployment_sha must be a lowercase 40-hex commit"
              exit 1
            }
            verify_dir=$(mktemp -d)
            trap 'rm -rf "$verify_dir"' EXIT
            git -C "$verify_dir" init --quiet
            git -C "$verify_dir" fetch --quiet --no-tags --depth=1 \
              "https://github.com/${GITHUB_REPOSITORY}.git" "$REQUESTED_SHA"
            deployment_sha=$(git -C "$verify_dir" rev-parse FETCH_HEAD)
            [ "$deployment_sha" = "$REQUESTED_SHA" ] || {
              echo "::error::Fetched commit does not match requested deployment_sha"
              exit 1
            }
          else
            deployment_sha=$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/heads/$BRANCH" | awk 'NR == 1 { print $1 }')
          fi
          [[ "$deployment_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve immutable deployment SHA for $BRANCH"; exit 1; }
          echo "deployment_sha=$deployment_sha" >> "$GITHUB_OUTPUT"
          echo "Resolved immutable deployment snapshot: $deployment_sha"

  deploy:
    name: Deploy worker to Hetzner host (${{ needs.determine-env.outputs.environment }} @ ${{ needs.determine-env.outputs.deployment_sha }})
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    needs: determine-env
    environment: ${{ needs.determine-env.outputs.environment }}
    timeout-minutes: 15
    env:
      SYSTEMD_UNIT: eliza-provisioning-worker.service
      DEPLOY_HOST: ${{ secrets.ELIZA_PROVISIONING_HOST }}
      DEPLOY_SSH_KEY: ${{ secrets.ELIZA_PROVISIONING_SSH_KEY }}
      DEPLOY_BRANCH: ${{ needs.determine-env.outputs.branch }}
      DEPLOY_SHA: ${{ needs.determine-env.outputs.deployment_sha }}
      # Headscale wiring for the provisioning worker (the CONSUMER of the
      # control plane). These are reconciled into /opt/eliza/cloud/.env.local
      # on every deploy so the box can never silently drift to a stale,
      # hand-edited headscale URL/key again. Reuse the canonical GH names
      # already established by arm-headscale-control-plane.yml.
      HEADSCALE_API_URL: ${{ vars.HEADSCALE_API_URL }}
      HEADSCALE_PUBLIC_URL: ${{ vars.HEADSCALE_PUBLIC_URL }}
      HEADSCALE_API_KEY: ${{ secrets.HEADSCALE_API_KEY }}
      # Base64 private key used by the daemon to reach dedicated-agent nodes.
      # Keep this in the protected environment instead of relying on a
      # hand-copied file that can drift or disappear when the CP host is rebuilt.
      CONTAINERS_SSH_KEY: ${{ secrets.CONTAINERS_SSH_KEY }}
      # Sandbox-registry Redis (TCP redis:// proxy, sandbox-reachable — never a
      # *.railway.internal host) so provisioned sandboxes self-register and
      # Discord/Telegram inbound routing works (#8621 / #8756). Reconciled into
      # /opt/eliza/cloud/.env.local the same way as HEADSCALE_*; when the secret
      # is unset the reconcile loop skips it and preserves any hand-set value,
      # so this is a no-op until the operator sets the GH secret.
      SANDBOX_REGISTRY_REDIS_URL: ${{ secrets.SANDBOX_REGISTRY_REDIS_URL }}
      # Cloud database DSN for the provisioning worker + agent router. Same
      # fallback chain as the migrator jobs (cloud-cf-deploy.yml /
      # cloud-deploy-backend.yml) so the CP daemon always resolves the exact
      # DB the Worker schema is migrated against. Before this, DATABASE_URL
      # only lived hand-edited on the box: when staging moved Neon → Railway
      # the CP silently kept polling the abandoned Neon DB for 3 weeks
      # (#15160). Reconciled into /opt/eliza/cloud/.env.local with the same
      # skip-when-unset semantics as HEADSCALE_* — an environment without any
      # of these secrets keeps its current hand-set value.
      DATABASE_URL: ${{ secrets.DATABASE_URL || secrets.RAILWAY_DATABASE_URL || secrets.NEON_DATABASE_URL }}
      # Field-encryption root secret used to unwrap per-org DEKs for encrypted
      # agent environment variables. Must match the Cloudflare Worker secret:
      # the Worker writes encrypted env vars, and this provisioning worker
      # decrypts them while creating the agent container (#15385). Skip-empty
      # reconcile semantics below preserve hand-set values until the secret is
      # wired for an environment.
      SECRETS_MASTER_KEY: ${{ secrets.SECRETS_MASTER_KEY }}
      # Operator pin for the canonical managed-agent image
      # (containersEnv.defaultAgentImageOverride; falls back to
      # ghcr.io/elizaos/eliza:stable when unset everywhere). This pin decides
      # which image every NEW provision and warm-pool replenish boots — it is
      # exactly the knob the #15228 crash-loop rollout needed. Before this it
      # lived ONLY hand-set on the CP box (the same drift class as
      # DATABASE_URL/#15160, tracked in the #15401 ledger): a CP re-arm lost
      # it silently. Set the GitHub environment VARIABLE (not secret — an
      # image ref is auditable config) to roll the fleet pin via deploy;
      # leave it unset to preserve whatever is hand-set on the box.
      ELIZA_AGENT_IMAGE: ${{ vars.ELIZA_AGENT_IMAGE }}
      # The edge Worker forwards the original per-agent host to this daemon.
      # Keep the daemon's parser on the same environment-specific suffix or a
      # staging host (`<uuid>.staging.elizacloud.ai`) is rejected as malformed.
      ELIZA_CLOUD_AGENT_BASE_DOMAIN: ${{ needs.determine-env.outputs.environment == 'production' && 'elizacloud.ai' || 'staging.elizacloud.ai' }}
    steps:
      - name: Check deploy configuration
        id: deploy_config
        run: |
          if [ -z "$DEPLOY_HOST" ] || [ -z "$DEPLOY_SSH_KEY" ]; then
            echo "configured=false" >> "$GITHUB_OUTPUT"
            echo "::warning::Missing ELIZA_PROVISIONING_HOST or ELIZA_PROVISIONING_SSH_KEY; skipping provisioning-worker deploy."
            {
              echo "### Provisioning worker deploy skipped"
              echo ""
              echo "Missing \`ELIZA_PROVISIONING_HOST\` or \`ELIZA_PROVISIONING_SSH_KEY\` for GitHub environment \`production\`."
              echo ""
              echo "Bootstrap the Hetzner host and secrets with:"
              echo ""
              echo "\`\`\`bash"
              echo "HCLOUD_TOKEN=... bun run packages/scripts/cloud/admin/bootstrap-provisioning-worker-host.mjs"
              echo "\`\`\`"
            } >> "$GITHUB_STEP_SUMMARY"
            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
              echo "::error::Manual deployment requires ELIZA_PROVISIONING_HOST and ELIZA_PROVISIONING_SSH_KEY."
              exit 1
            fi
            exit 0
          fi
          echo "configured=true" >> "$GITHUB_OUTPUT"

      - name: Ensure host prereqs (Node 24, swap, bunx symlink)
        if: steps.deploy_config.outputs.configured == 'true'
        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2
        with:
          host: ${{ env.DEPLOY_HOST }}
          username: deploy
          key: ${{ env.DEPLOY_SSH_KEY }}
          command_timeout: 5m
          script: |
            set -euo pipefail
            # Idempotent safety net for hosts provisioned BEFORE the cloud-init
            # template that pre-installs these. Each block is a no-op if already
            # set up — runs on every deploy so any host drift self-heals.

            # Node 24: the systemd unit calls `tsx` which spawns a Node process.
            # New cloud-init template installs it via NodeSource; older hosts
            # (eliza-staging-1 from the first TF apply) lack it.
            if ! command -v node >/dev/null 2>&1; then
              echo "[host-prereqs] installing Node 24"
              curl -fsSL https://deb.nodesource.com/setup_24.x | sudo bash -
              sudo apt-get install -y nodejs
            fi

            # Swap: smaller shapes (cpx22, cax11) OOM during `turbo run build`.
            # 8 GB swap once and forever; harmless on bigger shapes.
            if [ ! -f /swapfile ]; then
              echo "[host-prereqs] adding 8GB swap"
              sudo fallocate -l 8G /swapfile
              sudo chmod 600 /swapfile
              sudo mkswap /swapfile
              sudo swapon /swapfile
              grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab >/dev/null
            fi

            # bunx symlink: the canary `bun.sh/install` tarball stopped shipping
            # bunx as a separate binary at some point; build scripts that call
            # `bunx tsc` etc. then fail with `bun: command not found: bunx`.
            # `bunx` is just `bun` reading argv[0] — a symlink is sufficient.
            if [ ! -e /home/deploy/.bun/bin/bunx ] && [ -e /home/deploy/.bun/bin/bun ]; then
              echo "[host-prereqs] creating bunx symlink"
              ln -sf bun /home/deploy/.bun/bin/bunx
            fi

      - name: Deploy and restart worker
        if: steps.deploy_config.outputs.configured == 'true'
        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2
        with:
          host: ${{ env.DEPLOY_HOST }}
          username: deploy
          key: ${{ env.DEPLOY_SSH_KEY }}
          command_timeout: 10m
          envs: DEPLOY_BRANCH,DEPLOY_SHA,SYSTEMD_UNIT,HEADSCALE_API_URL,HEADSCALE_PUBLIC_URL,HEADSCALE_API_KEY,CONTAINERS_SSH_KEY,SANDBOX_REGISTRY_REDIS_URL,DATABASE_URL,SECRETS_MASTER_KEY,ELIZA_AGENT_IMAGE,ELIZA_CLOUD_AGENT_BASE_DOMAIN
          script: |
            set -euo pipefail
            echo "=== Deploying eliza-provisioning-worker (branch: $DEPLOY_BRANCH sha: $DEPLOY_SHA) ==="

            cd /opt/eliza
            export GIT_CONFIG_COUNT=1
            export GIT_CONFIG_KEY_0=safe.directory
            export GIT_CONFIG_VALUE_0=/opt/eliza

            rm -f .git/index.lock

            # A prior operator or deploy may have run git/install/build as
            # root, leaving either Git metadata or tracked source files owned
            # by root. Git then cannot reset or check out the immutable target
            # as `deploy`. The checkout is exclusively managed by this service,
            # so normalize the whole tree before any mutating Git command.
            sudo chown -R deploy:deploy /opt/eliza

            # Discard local drift, preserve env files. Quote globs so the
            # remote shell doesn't expand them before git sees the pattern,
            # and fail loud if clean errors out — silently swallowing it is
            # what let stale build artifacts survive and block checkout below.
            git reset --hard HEAD
            git clean -fdx \
              -e .env \
              -e .env.local \
              -e '.env.*' \
              -e cloud/.env.local \
              -e 'cloud/.env.*' \
              -e node_modules

            # Build artifacts (*.d.ts / *.d.ts.map) sometimes land next to
            # sources from a prior deploy; if the new HEAD tracks the same
            # paths, `git checkout -B` aborts with "untracked working tree
            # files would be overwritten". Sweep them before checkout.
            find packages -type f \( -name '*.d.ts' -o -name '*.d.ts.map' \) \
              ! -path '*/node_modules/*' -delete

            # bun.lock became tracked at 4ae092dcda; the host's previous deploy
            # may have left an untracked copy from `bun install` that pre-dates
            # the tracking change. `git clean -fdx` *should* remove it but
            # occasionally races with bun's lockfile rewrite, so force a delete
            # before the fetch + branch-checkout below.
            rm -f bun.lock

            # Use a refspec so a single-branch / shallow clone still creates
            # the remote-tracking ref for $DEPLOY_BRANCH on first deploy.
            #
            # --no-recurse-submodules on both fetch + checkout: the provisioning
            # worker daemon and agent router are pure Node/tsx code and never
            # touch the llama.cpp / opencode submodules. If the
            # deploy host has `fetch.recurseSubmodules=on-demand` (the default
            # when submodules are present), recursing through a superproject
            # commit whose submodule SHA was never pushed to the fork kills the
            # whole deploy with `upload-pack: not our ref`. Skip submodule
            # traversal entirely on this path.
            git -c fetch.recurseSubmodules=no fetch --no-recurse-submodules origin "$DEPLOY_SHA"
            git -c submodule.recurse=false checkout --no-recurse-submodules \
              -B "$DEPLOY_BRANCH" "$DEPLOY_SHA"
            test "$(git rev-parse HEAD)" = "$DEPLOY_SHA"

            # The `rm -f bun.lock` above deletes a TRACKED file; when bun.lock
            # is identical between the previous HEAD and the new tip, checkout
            # carries that deletion over as a local change and `bun install`
            # then resolves the whole tree fresh from package.json ranges —
            # newest-in-range @ai-sdk/* against the overridden provider-utils
            # pin took the worker down at boot (`does not provide an export
            # named 'secureJsonParse'`). Always restore the tracked lockfile so
            # installs are lockfile-pinned and deterministic.
            git checkout "$DEPLOY_SHA" -- bun.lock

            # git clean removes these intentionally ignored modules. Generate
            # and assert them before install/build, while the healthy daemons
            # are still running, so generator failure cannot cause downtime.
            bash packages/scripts/cloud/admin/ensure-generated-keywords.sh

            # Install the canary (Rust) Bun on first install.
            if ! command -v bun >/dev/null 2>&1; then
              curl -fsSL https://bun.sh/install | bash -s "canary"
              export BUN_INSTALL="$HOME/.bun"
              export PATH="$BUN_INSTALL/bin:$PATH"
            fi

            cd /opt/eliza
            for attempt in 1 2 3; do
              if bun install --no-save --ignore-scripts; then
                break
              fi
              if [ "$attempt" -eq 3 ]; then exit 1; fi
              echo "bun install attempt $attempt failed; retrying in 10s..."
              sleep 10
            done

            # The daemons run under Node/tsx rather than Bun. Node resolves
            # linked workspace packages through their built `node` exports, so
            # refresh linked package dist files after installing dependencies.
            #
            # The `git clean -fdx` above wipes `.turbo`, so every deploy does a
            # cold `build:core` (0 cached). Building the full graph includes
            # `@elizaos/plugin-local-inference`, whose tsc/bundler step exceeds
            # V8's default ~2 GB old-space and aborts the deploy with
            # `FATAL ERROR: ... JavaScript heap out of memory` (exit 134). Raise
            # the heap ceiling to match the repo's typecheck lane (8 GB).
            export NODE_OPTIONS="${NODE_OPTIONS:-} --max-old-space-size=8192"
            bun run build:core
            mkdir -p plugins/plugin-sql/node_modules/@elizaos
            rm -rf plugins/plugin-sql/node_modules/@elizaos/core
            ln -s ../../../../packages/core plugins/plugin-sql/node_modules/@elizaos/core
            bun run --cwd plugins/plugin-sql build

            # Defensive workspace symlinks: `bun install` should create these
            # automatically for any workspace package the root or a sibling
            # depends on, but the daemon (in packages/scripts/, which has NO
            # package.json and so is invisible to bun's workspace graph)
            # imports `@elizaos/cloud-shared` at runtime. On a fresh host the
            # symlink was missing — daemon then crashes with
            # `Cannot find package '@elizaos/cloud-shared'`. Recreate it here
            # idempotently after `bun install` is done.
            mkdir -p node_modules/@elizaos
            ln -sfn ../../packages/cloud/shared node_modules/@elizaos/cloud-shared

            # Install/refresh both systemd units (provisioning worker + agent router).
            sudo install -m 0644 \
              packages/scripts/cloud/admin/eliza-provisioning-worker.service \
              "/etc/systemd/system/$SYSTEMD_UNIT"
            sudo install -m 0644 \
              packages/scripts/cloud/admin/eliza-agent-router.service \
              /etc/systemd/system/eliza-agent-router.service
            sudo systemctl daemon-reload
            sudo systemctl enable "$SYSTEMD_UNIT" eliza-agent-router.service

            # Reconcile control-plane secrets (HEADSCALE_API_URL /
            # HEADSCALE_PUBLIC_URL / HEADSCALE_API_KEY /
            # SANDBOX_REGISTRY_REDIS_URL / DATABASE_URL / SECRETS_MASTER_KEY)
            # into the systemd EnvironmentFile so CI is the single source of
            # truth and the box self-heals on every deploy. The daemon +
            # cloud-shared read these straight from process.env
            # (headscale-client.ts, headscale-integration.ts,
            # docker-sandbox-provider.ts, the DB client, and encrypted
            # environment-variable decrypt), fed by
            # EnvironmentFile=/opt/eliza/cloud/.env.local. The
            # deploy's `git clean -e cloud/.env.local` preserves that file
            # verbatim, so before this these keys were only ever hand-edited on
            # the box and could drift to a stale headscale (the CP-box outage)
            # or a stale database (#15160: the CP polled an abandoned Neon DB
            # for 3 weeks after staging moved to Railway). Only rewrite a key
            # when the CI value is non-empty — an unset GH var/secret must
            # never blank a working box (a blank HEADSCALE_API_KEY makes
            # docker-sandbox-provider throw "HEADSCALE_API_KEY is not
            # configured" when a route is required).
            ENV_FILE=/opt/eliza/cloud/.env.local
            sudo touch "$ENV_FILE"
            # Pin this control-plane daemon to the AGENT lane. Unset, a single
            # daemon claims ALL job types (agent + apps); it would claim
            # APP_DEPLOY / CONTAINER_* jobs it can't run (no apps backend wired
            # here) and burn them down through retries. The dedicated apps
            # daemon (apps-provisioning-worker.ts) owns the apps lane. Reconciled
            # here so CI is the single source of truth and the box self-heals on
            # every deploy (same mechanism as the HEADSCALE_* keys below). This
            # value is constant — not a GH var — so it's reconciled directly,
            # never blanked.
            sudo sed -i "/^PROVISIONING_JOB_LANES=/d" "$ENV_FILE"
            printf 'PROVISIONING_JOB_LANES=%s\n' "agent" | sudo tee -a "$ENV_FILE" >/dev/null
            # Headscale is the required dedicated-agent ingress. Bridge-host
            # fallback was a legacy escape hatch; if it remains hand-set on the
            # box, docker-sandbox-provider deliberately disables VPN injection
            # and fresh staging agents never receive a headscale_ip (#15347).
            sudo sed -i "/^AGENT_ROUTER_ALLOW_BRIDGE_HOST_FALLBACK=/d" "$ENV_FILE"
            for kv in \
              "HEADSCALE_API_URL=$HEADSCALE_API_URL" \
              "HEADSCALE_PUBLIC_URL=$HEADSCALE_PUBLIC_URL" \
              "HEADSCALE_API_KEY=$HEADSCALE_API_KEY" \
              "CONTAINERS_SSH_KEY=$CONTAINERS_SSH_KEY" \
              "SANDBOX_REGISTRY_REDIS_URL=$SANDBOX_REGISTRY_REDIS_URL" \
              "DATABASE_URL=$DATABASE_URL" \
              "SECRETS_MASTER_KEY=$SECRETS_MASTER_KEY" \
              "ELIZA_AGENT_IMAGE=$ELIZA_AGENT_IMAGE" \
              "ELIZA_CLOUD_AGENT_BASE_DOMAIN=$ELIZA_CLOUD_AGENT_BASE_DOMAIN"; do
              key="${kv%%=*}"
              val="${kv#*=}"
              [ -n "$val" ] || continue
              sudo sed -i "/^${key}=/d" "$ENV_FILE"
              printf '%s=%s\n' "$key" "$val" | sudo tee -a "$ENV_FILE" >/dev/null
            done
            sudo chmod 600 "$ENV_FILE"

            sudo systemctl restart "$SYSTEMD_UNIT"
            sudo systemctl restart eliza-agent-router.service
            echo "=== Restart issued for both daemons ==="

      - name: Health check
        if: steps.deploy_config.outputs.configured == 'true'
        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2
        with:
          host: ${{ env.DEPLOY_HOST }}
          username: deploy
          key: ${{ env.DEPLOY_SSH_KEY }}
          command_timeout: 5m
          envs: SYSTEMD_UNIT,ELIZA_CLOUD_AGENT_BASE_DOMAIN
          script: |
            set -euo pipefail
            ENV_FILE=/opt/eliza/cloud/.env.local
            ACTUAL_AGENT_BASE_DOMAIN=$(sudo awk -F= '$1=="ELIZA_CLOUD_AGENT_BASE_DOMAIN" { value=$2 } END { print value }' "$ENV_FILE")
            if [ "$ACTUAL_AGENT_BASE_DOMAIN" != "$ELIZA_CLOUD_AGENT_BASE_DOMAIN" ]; then
              echo "::error::Agent router base-domain drift: expected $ELIZA_CLOUD_AGENT_BASE_DOMAIN, found ${ACTUAL_AGENT_BASE_DOMAIN:-<missing>}."
              exit 1
            fi
            # The worker logs to journald via systemd's stdout/stderr capture.
            # Info-level logs are filtered unless VERBOSE_LOGGING=true, so we
            # rely on systemd's own signals: process is active AND no fatal
            # error / restart-loop in the journal since deploy.
            HEALTH_SINCE_TS="$(date -u '+%Y-%m-%d %H:%M:%S')"
            STABLE_THRESHOLD_SEC=20
            FATAL_LOG_PATTERN="\[(provisioning-worker|agent-router)\] (fatal|unhandled rejection)|node:internal/.*Error:|Error \[ERR_|^[A-Za-z]+Error:"
            for attempt in $(seq 1 18); do
              if sudo systemctl is-active --quiet "$SYSTEMD_UNIT"; then
                JOURNAL=$(sudo journalctl -u "$SYSTEMD_UNIT" --since "$HEALTH_SINCE_TS" --no-pager 2>/dev/null || true)
                if echo "$JOURNAL" | grep -qE "$FATAL_LOG_PATTERN"; then
                  echo "::error::Worker logged a fatal error since deploy."
                  echo "$JOURNAL" | tail -n 50
                  exit 1
                fi
                # Active + uptime past the threshold = healthy. systemd's Restart=always
                # means a crashing process would resurrect; an uptime > threshold
                # therefore proves it stayed up across at least one poll cycle.
                UPTIME_SEC=$(systemctl show "$SYSTEMD_UNIT" --property=ActiveEnterTimestampMonotonic --value | awk '{ print int($1 / 1e6) }')
                NOW_SEC=$(awk '{print int($1)}' /proc/uptime)
                AGE=$(( NOW_SEC - UPTIME_SEC ))
                if [ "$AGE" -ge "$STABLE_THRESHOLD_SEC" ]; then
                  if ! sudo systemctl is-active --quiet eliza-agent-router.service; then
                    echo "Health check attempt $attempt/18: worker stable (${AGE}s), router not active yet."
                    sleep 5
                    continue
                  fi
                  ROUTER_JOURNAL=$(sudo journalctl -u eliza-agent-router.service --since "$HEALTH_SINCE_TS" --no-pager 2>/dev/null || true)
                  if echo "$ROUTER_JOURNAL" | grep -qE "$FATAL_LOG_PATTERN"; then
                    echo "::error::Agent router logged a fatal error since activation."
                    echo "$ROUTER_JOURNAL" | tail -n 50
                    exit 1
                  fi
                  ROUTER_UPTIME_SEC=$(systemctl show eliza-agent-router.service --property=ActiveEnterTimestampMonotonic --value | awk '{ print int($1 / 1e6) }')
                  ROUTER_AGE=$(( NOW_SEC - ROUTER_UPTIME_SEC ))
                  if [ "$ROUTER_AGE" -lt "$STABLE_THRESHOLD_SEC" ]; then
                    echo "Health check attempt $attempt/18: worker stable (${AGE}s), router only ${ROUTER_AGE}s old."
                    sleep 5
                    continue
                  fi
                  ROUTER_PORT=$(systemctl show eliza-agent-router.service --property=Environment --value | tr ' ' '\n' | awk -F= '$1=="AGENT_ROUTER_PORT" {print $2}')
                  : "${ROUTER_PORT:=3458}"
                  if curl -sf -m 3 "http://127.0.0.1:${ROUTER_PORT}/healthz" >/dev/null 2>&1; then
                    echo "Both daemons active and stable (worker ${AGE}s, router ${ROUTER_AGE}s, router /healthz OK) on attempt $attempt."
                    exit 0
                  fi
                  echo "Health check attempt $attempt/18: worker stable (${AGE}s), router ${ROUTER_AGE}s but /healthz not ready on port ${ROUTER_PORT}."
                  sleep 5
                  continue
                fi
                echo "Health check attempt $attempt/18: active but only ${AGE}s old, waiting for stability..."
              else
                echo "Health check attempt $attempt/18: not active yet."
              fi
              sleep 5
            done

            echo "::error::$SYSTEMD_UNIT failed to become healthy within 90s."
            sudo systemctl status "$SYSTEMD_UNIT" --no-pager || true
            sudo journalctl -u "$SYSTEMD_UNIT" -n 200 --no-pager || true
            sudo systemctl status eliza-agent-router.service --no-pager || true
            sudo journalctl -u eliza-agent-router.service -n 200 --no-pager || true
            exit 1

      - name: Notify Discord (Success)
        if: success() && steps.deploy_config.outputs.configured == 'true'
        uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708
        with:
          webhook: ${{ secrets.DISCORD_WEBHOOK }}
          ack_no_webhook: true
          title: "🚀 Eliza Provisioning Worker Deployed"
          nodetail: true
          description: |
            Branch: develop
            Commit: ${{ github.sha }}
          color: 0x00ff00

      - name: Notify Discord (Failure)
        if: failure() && steps.deploy_config.outputs.configured == 'true'
        uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708
        with:
          webhook: ${{ secrets.DISCORD_WEBHOOK }}
          ack_no_webhook: true
          title: "❌ Eliza Provisioning Worker Deploy Failed"
          nodetail: true
          description: |
            Branch: develop
            Commit: ${{ github.sha }}
          color: 0xff0000

# ci-touch: re-trigger prod daemon deploy (dispatch runs were runner-starved) 2026-06-21