#!/usr/bin/env bun /** * One-off Docker-disk prune for a single agent node — the immediate staging * unblock for a node that is OUT OF DISK (`no space left on device` while * pulling the agent image, breaking dedicated-agent provisioning on it). * * It runs the SAME reclamation the provisioning-worker daemon's disk-cleanup * cycle runs (`docker system prune -af` WITHOUT `--volumes` + clear stuck * containerd ingest from failed pulls + buildkit prune), over the SAME node-exec * primitive (`DockerSSHClient`), reading the daemon's configured env — so it * authenticates with `CONTAINERS_SSH_KEY` exactly like the daemon and never * touches key material itself. Prints `df` before and after; prints NO secrets. * * The recurring daemon cycle PREVENTS recurrence; this script UNBLOCKS NOW. * * ── Invocation on the staging CP ────────────────────────────────────────────── * The daemon runs on the control-plane host with its env at * `/opt/eliza/cloud/.env.local`. SSH in as the deploy user and run with bun from * the deployed `/opt/eliza` layout. The admin scripts are imported into that * layout, where the cloud-shared package resolves as `@elizaos/cloud-shared/...` * (NOT a relative `packages/cloud/shared/...` path — that only resolves in the * monorepo checkout). This script already imports via `@elizaos/cloud-shared/*` * so it resolves in both, but keep that in mind if you adapt it. * * ssh deploy@ # 167.233.105.184 (see infra access map) * cd /opt/eliza/cloud * set -a; . ./.env.local; set +a # load CONTAINERS_SSH_KEY etc. (no echo) * bun /opt/eliza/.../packages/scripts/cloud/admin/prune-node-disk.ts \ * --node-id eliza-core-95ea703e * * You can target by node-id (looked up in docker_nodes for hostname/port/user) or * pass --host directly to skip the DB lookup: * * bun .../prune-node-disk.ts --host 10.0.0.7 --ssh-user root * * Flags (env fallback in parens): * --node-id docker_nodes node_id to resolve (PRUNE_NODE_ID) * --host SSH target, skips the DB lookup (PRUNE_NODE_HOST) * --ssh-port SSH port (PRUNE_NODE_SSH_PORT) [default 22] * --ssh-user SSH user (PRUNE_NODE_SSH_USER) [default root] * --dry-run Print df + the planned reclamation, reclaim nothing. * * Exactly one of --node-id / --host is required. */ import * as path from "node:path"; import { fileURLToPath } from "node:url"; async function loadDeps() { const [{ dockerNodesRepository }, { DockerSSHClient }, diskMgr] = await Promise.all([ import("@elizaos/cloud-shared/db/repositories/docker-nodes"), import("@elizaos/cloud-shared/lib/services/docker-ssh"), import("@elizaos/cloud-shared/lib/services/node-disk-manager"), ]); return { dockerNodesRepository, DockerSSHClient, buildReclaimCommand: diskMgr.buildReclaimCommand, parseDfUsedPercent: diskMgr.parseDfUsedPercent, }; } // --------------------------------------------------------------------------- // Pure arg parsing (testable without the DB/SSH stack) // --------------------------------------------------------------------------- export interface PruneArgs { nodeId?: string; host?: string; sshPort: number; sshUser: string; dryRun: boolean; } export function parsePruneArgs( argv: string[], env: NodeJS.ProcessEnv, ): PruneArgs { const flags = new Map(); let dryRun = false; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === undefined) continue; if (arg === "--dry-run") { dryRun = true; continue; } if (arg.startsWith("--")) { const key = arg.slice(2); const value = argv[i + 1]; if (value === undefined || value.startsWith("--")) { throw new Error(`Flag --${key} requires a value`); } flags.set(key, value); i++; } } const nodeId = flags.get("node-id") ?? env.PRUNE_NODE_ID; const host = flags.get("host") ?? env.PRUNE_NODE_HOST; if (!nodeId && !host) { throw new Error( "Provide exactly one of --node-id (resolved via docker_nodes) or --host", ); } if (nodeId && host) { throw new Error("Provide only ONE of --node-id or --host, not both"); } const sshPort = Number.parseInt( flags.get("ssh-port") ?? env.PRUNE_NODE_SSH_PORT ?? "22", 10, ); if (!Number.isInteger(sshPort) || sshPort < 1 || sshPort > 65535) { throw new Error(`Invalid ssh-port: ${flags.get("ssh-port")}`); } const sshUser = flags.get("ssh-user") ?? env.PRUNE_NODE_SSH_USER ?? "root"; return { ...(nodeId ? { nodeId } : {}), ...(host ? { host } : {}), sshPort, sshUser, dryRun, }; } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- const DF_TIMEOUT_MS = 15_000; const RECLAIM_TIMEOUT_MS = 5 * 60_000; const DF_CMD = "df -P /var/lib/docker 2>/dev/null || df -P /"; async function main(): Promise { const args = parsePruneArgs(process.argv.slice(2), process.env); const { dockerNodesRepository, DockerSSHClient, buildReclaimCommand, parseDfUsedPercent, } = await loadDeps(); // Resolve SSH target: explicit --host, or the docker_nodes row for --node-id. let host = args.host; let sshPort = args.sshPort; let sshUser = args.sshUser; let fingerprint: string | undefined; if (args.nodeId) { const node = await dockerNodesRepository.findByNodeId(args.nodeId); if (!node) { throw new Error(`No docker_nodes row for node-id "${args.nodeId}"`); } host = node.hostname; sshPort = node.ssh_port ?? 22; sshUser = node.ssh_user ?? "root"; fingerprint = node.host_key_fingerprint ?? undefined; console.log( `[prune-node-disk] resolved node "${args.nodeId}" -> ${sshUser}@${host}:${sshPort}`, ); } else { console.log(`[prune-node-disk] target ${sshUser}@${host}:${sshPort}`); } if (!host) throw new Error("No SSH host resolved"); const ssh = DockerSSHClient.getClient(host, sshPort, fingerprint, sshUser); try { await ssh.connect(); const before = await ssh.exec(DF_CMD, DF_TIMEOUT_MS); console.log("\n[prune-node-disk] df BEFORE:"); console.log(before.trimEnd()); const beforePct = parseDfUsedPercent(before); if (args.dryRun) { console.log("\n[prune-node-disk] --dry-run: would run reclamation:"); console.log(` ${buildReclaimCommand()}`); console.log("[prune-node-disk] --dry-run: no changes made."); return; } console.log( "\n[prune-node-disk] reclaiming (prune + clear stuck ingest)...", ); await ssh.exec(buildReclaimCommand(), RECLAIM_TIMEOUT_MS); const after = await ssh.exec(DF_CMD, DF_TIMEOUT_MS); console.log("\n[prune-node-disk] df AFTER:"); console.log(after.trimEnd()); const afterPct = parseDfUsedPercent(after); if (beforePct !== null && afterPct !== null) { console.log( `\n[prune-node-disk] done: ${beforePct}% -> ${afterPct}% (reclaimed ${ beforePct - afterPct } points)`, ); } else { console.log("\n[prune-node-disk] done (could not parse df percent)."); } } finally { await ssh.disconnect().catch(() => {}); } } function isMainModule(): boolean { const entry = process.argv[1]; return entry ? path.resolve(entry) === fileURLToPath(import.meta.url) : false; } if (isMainModule()) { // Load the daemon's env the same way the daemons do, so CONTAINERS_SSH_KEY and // DATABASE_URL resolve from /opt/eliza/cloud/.env.local without re-export. // // `loadLocalEnv` resolves the project root as ../../.. from the dir of the URL // it's given. The daemons live in `.../scripts/cloud/admin/daemons/.ts` and // resolve to `packages/scripts` (where the deployed `.env.local` is loaded // from). This script lives one dir up (`.../scripts/cloud/admin/`), so handing // load-env `import.meta.url` directly would resolve to `packages/` — one level // too high. Point it at a real file INSIDE the daemons dir so the project-root // resolution is byte-for-byte the same as the daemon's. import("./daemons/shared/load-env") .then(({ loadLocalEnv }) => { loadLocalEnv( new URL("./daemons/provisioning-worker.ts", import.meta.url).href, ); }) .catch(() => { // load-env is best-effort: if the layout differs, env may already be set // by `set -a; . ./.env.local` per the header invocation. }) .finally(() => { main().catch((error) => { console.error( "[prune-node-disk] failed:", error instanceof Error ? error.message : String(error), ); process.exit(1); }); }); }