name: Cloud Deploy Backend # Source: cloud/.github/workflows/deploy-backend.yml in the original elizaOS/cloud repo. # # This is a manual-only legacy operations workflow. Canonical Cloudflare # Worker/Pages releases and their database migrations are owned by # cloud-cf-deploy.yml; dispatch this workflow only for an explicit standalone # migration or legacy VPS deployment. on: workflow_dispatch: inputs: environment: description: 'Environment to deploy' required: true default: 'staging' type: choice options: - staging - production deploy_legacy_vps: description: 'Run the legacy VPS deploy after migrations' required: false default: false type: boolean concurrency: group: cloud-deploy-backend-${{ github.ref }} cancel-in-progress: false env: BUN_VERSION: "canary" # Default to least privilege. Override per-job where needed. permissions: contents: read jobs: determine-env: name: Determine Environment runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} outputs: environment: ${{ steps.env.outputs.environment }} branch: ${{ steps.env.outputs.branch }} steps: - name: Set environment id: env run: | echo "environment=${{ github.event.inputs.environment }}" >> $GITHUB_OUTPUT echo "branch=${{ github.event.inputs.environment == 'production' && 'main' || 'develop' }}" >> $GITHUB_OUTPUT migrate-db: name: Run Database Migrations runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} needs: determine-env # Job-level concurrency groups are repo-wide: this serializes an explicit # legacy migration against the canonical migrate-db gate in # cloud-cf-deploy.yml (#11208). concurrency: group: cloud-db-migrate-${{ needs.determine-env.outputs.environment }} cancel-in-progress: false environment: ${{ needs.determine-env.outputs.environment }} timeout-minutes: 10 env: MIGRATION_DATABASE_URL: ${{ secrets.DATABASE_URL || secrets.RAILWAY_DATABASE_URL || secrets.NEON_DATABASE_URL }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 with: bun-version: ${{ env.BUN_VERSION }} - name: Install dependencies run: | for attempt in 1 2 3; do if [ "$attempt" -eq 3 ]; then bun install --no-save --ignore-scripts --verbose && exit 0 else bun install --no-save --ignore-scripts && exit 0 fi echo "bun install attempt $attempt failed; retrying in 10s..." sleep 10 done exit 1 - name: Fail fast when database secret is missing if: ${{ env.MIGRATION_DATABASE_URL == '' }} run: | # Previously this step quietly skipped migrations whenever the secret # was unset, so 46+ migrations silently accumulated against prod from # 2026-04-22 onward (sukimyfun Stripe webhook 500 RCA, 2026-05-20). # Fail loudly instead — a deploy that ships code expecting a newer # schema must not succeed if the DB can't be migrated to match. echo "::error::DATABASE_URL / RAILWAY_DATABASE_URL / NEON_DATABASE_URL secret is missing on the '${{ needs.determine-env.outputs.environment }}' environment. The schema cannot be migrated, so the worker would deploy against a stale DB." echo "Add the secret in: Settings → Environments → ${{ needs.determine-env.outputs.environment }} → Environment secrets." exit 1 - name: Run migrations env: DATABASE_URL: ${{ env.MIGRATION_DATABASE_URL }} run: | # The migrate entry imports @elizaos/core, whose i18n barrel pulls in the # gitignored generated keyword data. Source-mode (eliza-source) migrate # never builds core, so generate the file first (mirrors core's prebuild). node packages/shared/scripts/generate-keywords.mjs bun run db:cloud:migrate - name: Notify Discord (Success) if: success() uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 with: webhook: ${{ secrets.DISCORD_WEBHOOK }} ack_no_webhook: true title: "🗄️ Database Migrated" nodetail: true description: | Environment: ${{ needs.determine-env.outputs.environment }} Branch: ${{ needs.determine-env.outputs.branch }} Commit: ${{ github.sha }} color: 0x00ff00 - name: Notify Discord (Failure) if: failure() uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 with: webhook: ${{ secrets.DISCORD_WEBHOOK }} ack_no_webhook: true title: "❌ Database Migration Failed" nodetail: true description: | Environment: ${{ needs.determine-env.outputs.environment }} Branch: ${{ needs.determine-env.outputs.branch }} Commit: ${{ github.sha }} color: 0xff0000 deploy: name: Deploy to agent VPS if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_legacy_vps }} runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} needs: [determine-env, migrate-db] environment: ${{ needs.determine-env.outputs.environment }} timeout-minutes: 45 env: DEPLOY_VPS_HOST: ${{ secrets.AGENT_VPS_HOST || secrets.ELIZA_VPS_HOST }} DEPLOY_VPS_SSH_KEY: ${{ secrets.AGENT_VPS_SSH_KEY || secrets.ELIZA_VPS_SSH_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 with: bun-version: ${{ env.BUN_VERSION }} - name: Install dependencies run: | for attempt in 1 2 3; do if [ "$attempt" -eq 3 ]; then bun install --no-save --ignore-scripts --verbose && exit 0 else bun install --no-save --ignore-scripts && exit 0 fi echo "bun install attempt $attempt failed; retrying in 10s..." sleep 10 done exit 1 - name: Build backend artifact run: | bun run --cwd packages/cloud/api codegen bun run --cwd packages/cloud/api lint - name: Prepare SSH access run: | if [ -z "$DEPLOY_VPS_HOST" ] || [ -z "$DEPLOY_VPS_SSH_KEY" ]; then echo "::error::Missing VPS deploy secrets. Configure AGENT_VPS_HOST/AGENT_VPS_SSH_KEY or ELIZA_VPS_HOST/ELIZA_VPS_SSH_KEY for the GitHub environment." exit 1 fi install -d -m 700 "$HOME/.ssh" printf '%s\n' "$DEPLOY_VPS_SSH_KEY" > "$HOME/.ssh/id_ed25519" chmod 600 "$HOME/.ssh/id_ed25519" ssh-keyscan -H "$DEPLOY_VPS_HOST" >> "$HOME/.ssh/known_hosts" - name: Prepare VPS checkout uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 with: host: ${{ env.DEPLOY_VPS_HOST }} username: deploy key: ${{ env.DEPLOY_VPS_SSH_KEY }} script_stop: true command_timeout: 30m script: | set -e echo "=== Deploying eliza-cloud backend ===" sudo chown -R deploy:deploy /opt/eliza-cloud cd /opt/eliza-cloud # Use env-based config to avoid writing ~/.gitconfig (lock file failures) export GIT_CONFIG_COUNT=1 export GIT_CONFIG_KEY_0=safe.directory export GIT_CONFIG_VALUE_0=/opt/eliza-cloud # --- Disk cleanup (prevents "Out of diskspace" failures) --- echo "Disk before cleanup:" df -h / | tail -1 # Remove stale index.lock FIRST (previous failed git op leaves this) rm -f .git/index.lock # Remove previous build output and local build caches. # Keep node_modules in place so bun install can reuse the existing # tree instead of redownloading the full dependency graph onto a # nearly-full VPS. rm -rf .next-build .next .turbo node_modules/.cache # Drop all accumulated deploy stashes git stash clear 2>/dev/null || true # Prune loose git objects git prune 2>/dev/null || true # Trim Bun/Turbo caches aggressively export BUN_INSTALL="$HOME/.bun" export PATH="$BUN_INSTALL/bin:$PATH" rm -rf "$HOME/.bun/install/cache" "$HOME/.cache/bun" "$HOME/.cache/turbo" if command -v bun >/dev/null 2>&1; then bun pm cache rm 2>/dev/null || true fi # Free host-level disk before touching the checkout. The root # filesystem fills up outside /opt/eliza-cloud too, especially with # old Docker layers, package caches, and journals. sudo apt-get clean 2>/dev/null || true sudo journalctl --vacuum-size=500M 2>/dev/null || true sudo docker system prune -af 2>/dev/null || true sudo rm -rf /root/.bun/install/cache /root/.cache/bun /root/.cache/turbo 2>/dev/null || true # Discard local checkout drift before fetch. Cleaning untracked # files first frees space for git to rewrite tracked files. # Preserve the server's environment file and keep dependency trees # if there is enough free disk to reuse them. echo "Discarding local checkout drift..." git clean -fdx \ -e .env \ -e .env.local \ -e .env.* \ -e node_modules \ -e services/gateway-discord/node_modules \ -e services/gateway-webhook/node_modules \ -e packages/ui/node_modules \ -e services/agent-server/node_modules 2>/dev/null || true rm -f .git/index.lock git checkout -- . 2>/dev/null || true rm -f .git/index.lock # The systemd unit on the VPS expects a repo-local env file, but the # exact filename can vary across older hosts. Preserve all root env # files and recreate the common aliases from whichever one survived. ENV_SOURCE="" for candidate in .env.local .env.production .env.staging .env; do if [ -f "$candidate" ]; then ENV_SOURCE="$candidate" break fi done if [ -n "$ENV_SOURCE" ]; then for alias in .env .env.local .env.production .env.staging; do if [ ! -e "$alias" ]; then ln -s "$ENV_SOURCE" "$alias" fi done fi echo "Env files available on VPS checkout:" ls -la .env* 2>/dev/null || true AVAILABLE_KB="$(df --output=avail -k / | tail -1 | tr -d ' ')" if [ "${AVAILABLE_KB:-0}" -lt 5000000 ]; then echo "Still under 5GB free; removing workspace node_modules as a fallback..." rm -rf \ node_modules \ services/gateway-discord/node_modules \ services/gateway-webhook/node_modules \ packages/ui/node_modules \ services/agent-server/node_modules fi echo "Disk after cleanup:" df -h / | tail -1 # --- End disk cleanup --- # Fetch and checkout BRANCH="${{ needs.determine-env.outputs.branch }}" git fetch origin "$BRANCH" git checkout -B "$BRANCH" "origin/$BRANCH" # Install runtime dependencies and regenerate lightweight assets. if ! command -v bun >/dev/null 2>&1; then curl -fsSL https://bun.sh/install | bash export PATH="$BUN_INSTALL/bin:$PATH" fi for attempt in 1 2 3; do if bun install --no-save --ignore-scripts; then break fi if [ "$attempt" -eq 3 ]; then exit 1 fi echo "bun install attempt $attempt failed; retrying in 10s..." sleep 10 done bun run generate:llms # The build now runs on GitHub Actions to avoid ENOSPC during # `next build` on the VPS. Clear any stale output and leave an empty # destination for the streamed artifact. rm -rf "$HOME/.bun/install/cache" "$HOME/.cache/bun" rm -rf node_modules/.cache rm -rf .next-build mkdir -p .next-build - name: Upload prebuilt Next output run: | # Only ship the runtime output. The webpack cache dominates .next-build # size locally but is not needed by `next start` on the VPS. tar -C .next-build -czf - \ BUILD_ID \ app-path-routes-manifest.json \ build-manifest.json \ export-marker.json \ images-manifest.json \ next-minimal-server.js.nft.json \ next-server.js.nft.json \ package.json \ prerender-manifest.json \ react-loadable-manifest.json \ required-server-files.js \ required-server-files.json \ routes-manifest.json \ server \ static | ssh -i "$HOME/.ssh/id_ed25519" \ deploy@"$DEPLOY_VPS_HOST" \ 'set -e; cd /opt/eliza-cloud; rm -rf .next .next-build/*; tar -xzf - -C .next-build; ln -sfn .next-build .next' - name: Restart services uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 with: host: ${{ env.DEPLOY_VPS_HOST }} username: deploy key: ${{ env.DEPLOY_VPS_SSH_KEY }} script_stop: true script: | set -e cd /opt/eliza-cloud rm -rf .next ln -sfn .next-build .next if ! sudo systemctl restart eliza-cloud; then sudo systemctl cat eliza-cloud || true sudo systemctl show eliza-cloud \ --property=FragmentPath,EnvironmentFiles,ExecStart,WorkingDirectory || true ls -la /opt/eliza-cloud/.env* 2>/dev/null || true sudo systemctl status eliza-cloud --no-pager || true sudo journalctl -u eliza-cloud -n 100 --no-pager || true exit 1 fi sudo systemctl restart agent-provisioning-worker echo "=== Deploy complete ===" - name: Health Check uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 with: host: ${{ env.DEPLOY_VPS_HOST }} username: deploy key: ${{ env.DEPLOY_VPS_SSH_KEY }} script_stop: true script: | echo "Checking eliza-cloud health..." for attempt in $(seq 1 18); do for url in http://localhost:3000/api/health http://localhost:3334/api/health; do if curl -sf "$url" > /dev/null; then echo "Health check passed on attempt $attempt via $url." exit 0 fi done echo "Health check attempt $attempt/18 failed; retrying in 5s..." sleep 5 done echo "eliza-cloud failed health checks after 90s. Collecting diagnostics..." sudo ss -ltnp | grep -E ':(3000|3334)\\s' || true sudo systemctl status eliza-cloud --no-pager || true sudo journalctl -u eliza-cloud -n 100 --no-pager || true exit 1 - name: Notify Discord if: success() uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 with: webhook: ${{ secrets.DISCORD_WEBHOOK }} ack_no_webhook: true title: "🚀 Backend Deployed" description: | Environment: ${{ needs.determine-env.outputs.environment }} Branch: ${{ needs.determine-env.outputs.branch }} Commit: ${{ github.sha }} color: 0x00ff00 - name: Notify Discord (Failure) if: failure() uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 with: webhook: ${{ secrets.DISCORD_WEBHOOK }} ack_no_webhook: true title: "❌ Backend Deploy Failed" description: | Environment: ${{ needs.determine-env.outputs.environment }} Branch: ${{ needs.determine-env.outputs.branch }} Commit: ${{ github.sha }} color: 0xff0000