#!/usr/bin/env bun /** * Onboard an EXISTING host (e.g. a Hetzner robot box) as an elizaOS Cloud * Docker node — with zero manual SSH/DB steps. * * A robot/auctioned host can't be cloud-init'd (it's already running), so this * script runs the bootstrap-equivalent steps over SSH and then registers the * node into `docker_nodes` the same way the autoscaler / bootstrap-callback do. * It is the operator-side counterpart to `buildContainerNodeUserData`. * * Every step is idempotent and safe to re-run: * 1. verify/install Docker + ensure the daemon is running, * 2. ensure the shared bridge network exists, * 3. ensure deterministic ghcr access — THE robot fix: clear any stale * stored credential (an expired ghcr token in /root/.docker/config.json * overrides anonymous access and bricks the public-image pull with * `denied`). Reuses `ensureRegistryAccess`. * 4. clean zombie/stale agent containers (exited/created orphans matching the * agent naming scheme — never an active sandbox), * 5. upsert the node into `docker_nodes` (update if it already exists), * 6. print a clear summary of what changed vs. was already in place. * * No secrets are hard-coded: the registry token (if any) comes from the * control-plane env via `containersEnv`; the DB target from `DATABASE_URL`. * * Usage: * DATABASE_URL=... bun run packages/scripts/cloud/admin/onboard-docker-node.ts \ * --host 1.2.3.4 --key ~/.ssh/id_ed25519_eliza --node-id robot-fsn1-01 * * Flags (env fallback in parens): * --host SSH target (ONBOARD_NODE_HOST) [required] * --node-id Logical node id (ONBOARD_NODE_ID) [required] * --key SSH private key path (ONBOARD_NODE_SSH_KEY) [default ~/.ssh/id_ed25519] * --ssh-port SSH port (ONBOARD_NODE_SSH_PORT) [default 22] * --ssh-user SSH user (ONBOARD_NODE_SSH_USER) [default root] * --capacity Agent capacity (ONBOARD_NODE_CAPACITY) [default 8] * --dry-run Print the planned steps, touch nothing. */ import * as os from "node:os"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; // The cloud-shared modules are imported lazily inside main() (see loadDeps) so // importing this file for its pure helpers — e.g. from the unit test — does not // drag in the Drizzle / plugin-sql DB stack. async function loadDeps() { const [ { dockerNodesRepository, stampDockerNodeEnvironmentMetadata }, { ensureRegistryAccess }, dockerUtils, { DockerSSHClient }, ] = await Promise.all([ import("@elizaos/cloud-shared/db/repositories/docker-nodes"), import( "@elizaos/cloud-shared/lib/services/containers/hetzner-client/registry" ), import("@elizaos/cloud-shared/lib/services/docker-sandbox-utils"), import("@elizaos/cloud-shared/lib/services/docker-ssh"), ]); return { dockerNodesRepository, stampDockerNodeEnvironmentMetadata, ensureRegistryAccess, buildEnsureNetworkCmd: dockerUtils.buildEnsureNetworkCmd, shellQuote: dockerUtils.shellQuote, DockerSSHClient, }; } // --------------------------------------------------------------------------- // Pure helpers (unit-tested in onboard-docker-node.test.ts) // --------------------------------------------------------------------------- /** Container-name prefixes the cloud control plane uses for agent workloads. */ export const AGENT_CONTAINER_PREFIXES = ["agent-", "cloud-container-"] as const; /** Docker states that mean a container is NOT actively serving — safe to reap. */ const REAPABLE_STATES = ["exited", "created", "dead"] as const; export interface DockerPsRow { name: string; state: string; } /** * Parse the output of `docker ps -a --format '{{.Names}}\t{{.State}}'`. * Tolerant of blank lines and trailing whitespace. */ export function parseDockerPs(output: string): DockerPsRow[] { return output .split("\n") .map((line) => line.trim()) .filter((line) => line.length > 0 && !line.startsWith("[stderr]")) .map((line) => { const [name, state] = line.split("\t"); return { name: (name ?? "").trim(), state: (state ?? "").trim().toLowerCase(), }; }) .filter((row) => row.name.length > 0); } /** * Conservative zombie filter: an agent-named container in a non-running state. * Running / restarting / paused containers are NEVER selected, so an active * sandbox is never touched even if its DB row drifted. */ export function selectZombieAgentContainers(rows: DockerPsRow[]): string[] { return rows .filter( (row) => AGENT_CONTAINER_PREFIXES.some((prefix) => row.name.startsWith(prefix), ) && (REAPABLE_STATES as readonly string[]).includes(row.state), ) .map((row) => row.name); } export interface OnboardArgs { host: string; nodeId: string; keyPath: string; sshPort: number; sshUser: string; capacity: number; dryRun: boolean; } interface ExistingDockerNodePin { host_key_fingerprint: string | null; capacity: number; } interface OnboardSshConfig { hostname: string; port: number; username: string; privateKeyPath: string; hostKeyFingerprint?: string; onHostKeyDiscovered: (hostname: string, fingerprint: string) => Promise; } export function buildOnboardSshConfig( args: OnboardArgs, existing: ExistingDockerNodePin | null, onHostKeyDiscovered: OnboardSshConfig["onHostKeyDiscovered"], ): OnboardSshConfig { return { hostname: args.host, port: args.sshPort, username: args.sshUser, privateKeyPath: args.keyPath, hostKeyFingerprint: existing?.host_key_fingerprint ?? undefined, onHostKeyDiscovered, }; } export function hostKeyFingerprintForOnboardUpsert( existing: ExistingDockerNodePin | null, capturedFingerprint: string | undefined, ): string | null { return existing?.host_key_fingerprint ?? capturedFingerprint ?? null; } /** * Capacity to write on (re-)onboard. Once a node exists, its slot count is * operator-owned (tuned via the admin PATCH route or a direct DB update to * match the box's real RAM), so a re-onboard preserves it and never resets it * to the `--capacity` default (which is sized for the small cpx32-class node * this script was born on). The flag value is only used to seed a brand-new * row. */ export function capacityForOnboardUpsert( existing: ExistingDockerNodePin | null, flagCapacity: number, ): number { return existing?.capacity ?? flagCapacity; } /** Parse argv + env into a validated config. Throws on missing required fields. */ export function parseArgs(argv: string[], env: NodeJS.ProcessEnv): OnboardArgs { const flags = new Map(); let dryRun = false; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === undefined) continue; if (arg === "--dry-run") { dryRun = true; continue; } if (arg.startsWith("--")) { const key = arg.slice(2); const value = argv[i + 1]; if (value === undefined || value.startsWith("--")) { throw new Error(`Flag --${key} requires a value`); } flags.set(key, value); i++; } } const host = flags.get("host") ?? env.ONBOARD_NODE_HOST; const nodeId = flags.get("node-id") ?? env.ONBOARD_NODE_ID; if (!host) throw new Error("Missing --host (or ONBOARD_NODE_HOST)"); if (!nodeId) throw new Error("Missing --node-id (or ONBOARD_NODE_ID)"); const keyPath = flags.get("key") ?? env.ONBOARD_NODE_SSH_KEY ?? path.join(os.homedir(), ".ssh", "id_ed25519"); const sshPort = Number.parseInt( flags.get("ssh-port") ?? env.ONBOARD_NODE_SSH_PORT ?? "22", 10, ); const sshUser = flags.get("ssh-user") ?? env.ONBOARD_NODE_SSH_USER ?? "root"; const capacityRaw = flags.get("capacity") ?? env.ONBOARD_NODE_CAPACITY ?? "8"; const capacity = Number.parseInt(capacityRaw, 10); if (!Number.isInteger(sshPort) || sshPort < 1 || sshPort > 65535) { throw new Error( `Invalid ssh-port: ${flags.get("ssh-port") ?? env.ONBOARD_NODE_SSH_PORT}`, ); } if (!Number.isInteger(capacity) || capacity < 1 || capacity > 64) { throw new Error(`Invalid capacity (must be 1..64): ${capacityRaw}`); } return { host, nodeId, keyPath, sshPort, sshUser, capacity, dryRun }; } // --------------------------------------------------------------------------- // Onboarding flow // --------------------------------------------------------------------------- async function main(): Promise { const args = parseArgs(process.argv.slice(2), process.env); // Resolve network/image from config for the preview. `containersEnv` is a // light import; the heavier DB/SSH stack is only loaded once we commit to // touching the host, so --dry-run stays side-effect-free. const { containersEnv } = await import( "@elizaos/cloud-shared/lib/config/containers-env" ); const network = containersEnv.dockerNetwork(); const image = containersEnv.defaultAgentImage(); console.log( `[onboard] target ${args.sshUser}@${args.host}:${args.sshPort} as node "${args.nodeId}"`, ); console.log( `[onboard] network=${network} image=${image} capacity=${args.capacity}`, ); if (args.dryRun) { console.log("[onboard] --dry-run: no changes will be made."); return; } const { dockerNodesRepository, stampDockerNodeEnvironmentMetadata, ensureRegistryAccess, buildEnsureNetworkCmd, shellQuote, DockerSSHClient, } = await loadDeps(); const summary: string[] = []; const existing = await dockerNodesRepository.findByNodeId(args.nodeId); // Re-onboard must verify against the stored pin before any root SSH command // runs. Only a never-pinned node takes the TOFU branch and persists the // captured key during the upsert below. let capturedFingerprint: string | undefined; const ssh = new DockerSSHClient( buildOnboardSshConfig(args, existing, async (hostname, fingerprint) => { capturedFingerprint = fingerprint; console.log( `[onboard] TOFU captured host key for ${hostname}: SHA256:${fingerprint}`, ); }), ); try { // 1. Docker present + running (install via get.docker.com only if missing). const hasDocker = await ssh .exec("command -v docker >/dev/null 2>&1 && echo yes || echo no", 30_000) .then((out) => out.includes("yes")); if (!hasDocker) { console.log("[onboard] docker not found — installing via get.docker.com"); await ssh.exec("curl -fsSL https://get.docker.com | sh", 5 * 60 * 1000); summary.push("installed Docker"); } else { summary.push("Docker already present"); } await ssh.exec( "systemctl enable --now docker >/dev/null 2>&1 || true", 60_000, ); await ssh.exec("docker info >/dev/null 2>&1", 30_000); summary.push("Docker daemon running"); // 2. Shared bridge network (idempotent, race-safe). await ssh.exec(buildEnsureNetworkCmd(network), 30_000); summary.push(`network "${network}" ensured`); // 3. THE robot fix: deterministic registry access (clear stale ghcr cred). await ensureRegistryAccess(ssh, image); summary.push( containersEnv.registryToken() || containersEnv.registryTokenFile() ? "ghcr login refreshed (token configured)" : "ghcr stale creds cleared (anonymous pull)", ); // 4. Reap zombie agent containers (orphaned, non-running). Conservative. const psOutput = await ssh.exec( "docker ps -a --format '{{.Names}}\t{{.State}}'", 30_000, ); const zombies = selectZombieAgentContainers(parseDockerPs(psOutput)); if (zombies.length > 0) { await ssh.exec( `docker rm -f ${zombies.map(shellQuote).join(" ")}`, 60_000, ); summary.push( `removed ${zombies.length} zombie container(s): ${zombies.join(", ")}`, ); } else { summary.push("no zombie containers"); } // 5. Pull the agent image now so the first deploy on this node is warm. console.log( `[onboard] pre-pulling ${image} (first run can take a few minutes)`, ); await ssh .exec(`docker pull ${shellQuote(image)}`, 10 * 60 * 1000) .then(() => summary.push("agent image pulled")) .catch((err) => { console.warn( `[onboard] image pre-pull failed (node still registers; will retry on deploy): ${err instanceof Error ? err.message : String(err)}`, ); summary.push("agent image pre-pull FAILED (non-fatal)"); }); // 6. Register / upsert into docker_nodes — same shape as bootstrap-callback. if (existing) { await dockerNodesRepository.update(existing.id, { hostname: args.host, ssh_port: args.sshPort, ssh_user: args.sshUser, // Preserve an operator-tuned capacity across re-onboards; the // `--capacity` default only seeds a brand-new row (see create branch). capacity: capacityForOnboardUpsert(existing, args.capacity), status: "unknown", // Never overwrite an established pin during re-onboard; a differing // presented key must fail in DockerSSHClient before this update. host_key_fingerprint: hostKeyFingerprintForOnboardUpsert( existing, capturedFingerprint, ), metadata: stampDockerNodeEnvironmentMetadata({ ...((existing.metadata as Record) ?? {}), provider: "operator-onboarded", lastOnboardedAt: new Date().toISOString(), }), }); summary.push(`docker_nodes row updated (${args.nodeId})`); } else { await dockerNodesRepository.create({ node_id: args.nodeId, hostname: args.host, ssh_port: args.sshPort, ssh_user: args.sshUser, capacity: args.capacity, enabled: true, status: "unknown", allocated_count: 0, // Persist the TOFU-captured pin so later control-plane SSH is verified. host_key_fingerprint: hostKeyFingerprintForOnboardUpsert( null, capturedFingerprint, ), metadata: stampDockerNodeEnvironmentMetadata({ provider: "operator-onboarded", onboardedAt: new Date().toISOString(), }), }); summary.push(`docker_nodes row created (${args.nodeId})`); } } finally { // error-policy:J6 best-effort SSH teardown; the onboarding result already stands await ssh.disconnect().catch(() => {}); } console.log("\n[onboard] done:"); for (const line of summary) console.log(` - ${line}`); } function isMainModule(): boolean { const entry = process.argv[1]; return entry ? path.resolve(entry) === fileURLToPath(import.meta.url) : false; } if (isMainModule()) { main().catch((error) => { console.error( "[onboard] failed:", error instanceof Error ? error.message : String(error), ); process.exit(1); }); }