elizaos--eliza/packages/scripts/sync-artifacts.mjs

#!/usr/bin/env node
// Drives repo automation sync artifacts with explicit CLI and CI behavior.
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { once } from "node:events";
/**
 * sync-artifacts.mjs
 *
 * Large generated / downloadable artifacts (benchmark fixtures, CAD exports,
 * build outputs, vendored binaries, media) are NOT committed to this repo —
 * they would make `git clone` slow and heavy. They live as a single bundle on
 * the elizaOS/eliza-archive release and are pulled here on install.
 *
 * This script is idempotent: it no-ops when the on-disk artifact version
 * already matches packages/scripts/artifacts-manifest.json.
 *
 *   bun packages/scripts/sync-artifacts.mjs        # used by postinstall
 *   ELIZA_SKIP_ARTIFACT_SYNC=1 ...                 # skip (CI lanes that don't need them)
 *
 * On download failure it warns and exits 0 so a network blip never blocks
 * `bun install`; re-run `bun run sync:artifacts` to retry.
 */
import {
  createWriteStream,
  existsSync,
  readdirSync,
  readFileSync,
  rmSync,
  statSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
const MANIFEST = join(ROOT, "packages", "scripts", "artifacts-manifest.json");
const MARKER = join(ROOT, ".eliza-artifacts-version");
const log = (m) => console.log(`[sync-artifacts] ${m}`);
const warn = (m) => console.warn(`[sync-artifacts] WARNING: ${m}`);
const PROGRESS_INTERVAL_MS = 5000;
const STALE_TMP_MAX_AGE_MS = 6 * 60 * 60_000;

if (process.env.ELIZA_SKIP_ARTIFACT_SYNC === "1") {
  log("skipped (ELIZA_SKIP_ARTIFACT_SYNC=1)");
  process.exit(0);
}
if (!existsSync(MANIFEST)) {
  log("no artifacts-manifest.json; nothing to sync");
  process.exit(0);
}

const m = JSON.parse(readFileSync(MANIFEST, "utf8"));
const { version, asset } = m;
cleanupStaleTempArchives();

if (existsSync(MARKER) && readFileSync(MARKER, "utf8").trim() === version) {
  log(`artifacts already at ${version}; nothing to do`);
  process.exit(0);
}

const url =
  asset.url ||
  `https://github.com/${asset.repo}/releases/download/${asset.tag}/${asset.name}`;
const tmp = join(tmpdir(), `eliza-artifacts-${process.pid}.tar.gz`);

function formatBytes(bytes) {
  if (!Number.isFinite(bytes) || bytes <= 0) return "0 B";
  const units = ["B", "KiB", "MiB", "GiB"];
  let value = bytes;
  let unit = units[0];
  for (let i = 1; i < units.length && value >= 1024; i++) {
    value /= 1024;
    unit = units[i];
  }
  return `${value >= 10 || unit === "B" ? value.toFixed(0) : value.toFixed(1)} ${unit}`;
}

function formatDuration(ms) {
  if (!Number.isFinite(ms) || ms < 0) return "unknown";
  const seconds = Math.ceil(ms / 1000);
  if (seconds < 60) return `${seconds}s`;
  const minutes = Math.floor(seconds / 60);
  const remainder = seconds % 60;
  return `${minutes}m ${remainder}s`;
}

function progressStatus(received, total, startedAt) {
  const elapsedMs = Math.max(Date.now() - startedAt, 1);
  const bytesPerSecond = received / (elapsedMs / 1000);
  const parts = [`downloaded ${formatBytes(received)}`];
  if (total > 0) {
    const percent = Math.min((received / total) * 100, 100);
    const remainingBytes = Math.max(total - received, 0);
    const etaMs =
      bytesPerSecond > 0 ? (remainingBytes / bytesPerSecond) * 1000 : NaN;
    parts.push(
      `of ${formatBytes(total)}`,
      `(${percent.toFixed(1)}%)`,
      `eta ${formatDuration(etaMs)}`,
    );
  }
  parts.push(`at ${formatBytes(bytesPerSecond)}/s`);
  return parts.join(" ");
}

function cleanupStaleTempArchives() {
  const now = Date.now();
  let removed = 0;
  for (const entry of readdirSync(tmpdir())) {
    if (!/^eliza-artifacts-\d+\.tar\.gz$/.test(entry)) continue;
    const file = join(tmpdir(), entry);
    let stat;
    try {
      stat = statSync(file);
    } catch (err) {
      // error-policy:J6 best-effort temp cleanup; a racing process may remove the file first.
      warn(`could not stat stale temp archive ${file}: ${err.message}`);
      continue;
    }
    if (!stat.isFile()) continue;
    if (now - stat.mtimeMs < STALE_TMP_MAX_AGE_MS) continue;
    try {
      rmSync(file, { force: true });
      removed += 1;
    } catch (err) {
      // error-policy:J6 best-effort temp cleanup; failed cleanup must not block install.
      warn(`could not remove stale temp archive ${file}: ${err.message}`);
    }
  }
  if (removed > 0) {
    log(
      `removed ${removed} stale artifact temp archive${removed === 1 ? "" : "s"}`,
    );
  }
}

async function streamToFileWithProgress(response, dest, expectedBytes) {
  const headerBytes = Number(response.headers.get("content-length")) || 0;
  const totalBytes = headerBytes || expectedBytes || 0;
  const writer = createWriteStream(dest);
  const reader = response.body.getReader();
  const startedAt = Date.now();
  let received = 0;
  let lastLogAt = startedAt;
  let writerError;
  const streamError = new Promise((resolve) => {
    writer.once("error", (err) => {
      writerError = err;
      resolve();
    });
  });

  if (totalBytes > 0) {
    log(
      `artifact bundle size: ${formatBytes(totalBytes)}${m.fileCount ? ` across ${m.fileCount} files` : ""}`,
    );
  } else {
    log(
      "artifact bundle size unknown; reporting downloaded bytes until complete",
    );
  }

  try {
    while (true) {
      const { done, value } = await reader.read();
      if (done) break;
      received += value.byteLength;
      if (!writer.write(value)) {
        await Promise.race([once(writer, "drain"), streamError]);
        if (writerError) throw writerError;
      }
      const now = Date.now();
      if (now - lastLogAt >= PROGRESS_INTERVAL_MS) {
        log(progressStatus(received, totalBytes, startedAt));
        lastLogAt = now;
      }
    }
  } finally {
    reader.releaseLock();
  }

  await Promise.race([
    new Promise((resolve, reject) => {
      writer.end((err) => (err ? reject(err) : resolve()));
    }),
    streamError,
  ]);
  if (writerError) throw writerError;
  log(`download complete: ${progressStatus(received, totalBytes, startedAt)}`);
}

async function download(dest) {
  for (let attempt = 1; attempt <= 4; attempt++) {
    try {
      log(`downloading ${url} (attempt ${attempt}/4)`);
      const res = await fetch(url, { redirect: "follow" });
      if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
      await streamToFileWithProgress(res, dest, Number(asset.bytes) || 0);
      return true;
    } catch (err) {
      warn(`download failed: ${err.message}`);
      try {
        rmSync(dest, { force: true });
      } catch {}
      if (attempt < 4) await new Promise((r) => setTimeout(r, attempt * 2000));
    }
  }
  return false;
}

function sha256(file) {
  const h = createHash("sha256");
  h.update(readFileSync(file));
  return h.digest("hex");
}

const ok = await download(tmp);
if (!ok) {
  warn(`could not download artifact bundle after retries.`);
  warn(`the repo is usable, but large fixtures/binaries are absent.`);
  warn(`retry later with:  bun run sync:artifacts`);
  process.exit(0);
}

if (asset.sha256) {
  log(`verifying sha256 for ${formatBytes(asset.bytes || 0)} artifact bundle`);
  const got = sha256(tmp);
  if (got !== asset.sha256) {
    warn(`sha256 mismatch (got ${got}, want ${asset.sha256}); not extracting.`);
    rmSync(tmp, { force: true });
    process.exit(0);
  }
}

log(
  `extracting artifact bundle at repo root${m.fileCount ? ` (${m.fileCount} files)` : ""}…`,
);
// Prefer the Windows system bsdtar (System32\tar.exe): a GNU tar that may be
// first on PATH (Git-for-Windows / MSYS) misreads a `C:\...` archive path as an
// rsh `host:path` and dies with "Cannot connect to C: resolve failed". bsdtar
// (shipped with Windows 10 1803+/11) handles drive-letter paths natively.
// Like the download step above, never let extraction failure block `bun install`.
const tarBin =
  process.platform === "win32"
    ? join(process.env.SystemRoot || "C:\\Windows", "System32", "tar.exe")
    : "tar";
try {
  execFileSync(tarBin, ["-xzf", tmp, "-C", ROOT], { stdio: "inherit" });
  writeFileSync(MARKER, `${version}\n`);
  log(`done — artifacts synced to ${version}`);
} catch (err) {
  warn(`extraction failed: ${err.message}`);
  warn(`the repo is usable, but large fixtures/binaries are absent.`);
  warn(`retry later with:  bun run sync:artifacts`);
} finally {
  try {
    rmSync(tmp, { force: true });
  } catch {}
}